From 6ccdf2dbd0963247ce0d235fa7402449968ca598 Mon Sep 17 00:00:00 2001 From: blaipr Date: Mon, 26 Oct 2026 03:11:11 +0100 Subject: [PATCH] feat: connectors and item files --- CHANGELOG.md | 52 + CLAUDE.md | 28 +- README.md | 10 +- config/api_keys.yaml | 15 - config/api_keys/example.yaml | 17 + config/config.yaml | 4 +- config/connectors/granular.yaml | 16 + config/connectors/sample-infra.yaml | 6 + config/connectors/sample-inventory.yaml | 14 + config/connectors/ssh-facts.yaml | 10 + config/credentials.yaml | 24 - config/credentials/cred-github-token.yaml | 8 + config/credentials/cred-section9-api.yaml | 11 + config/credentials/cred-ssh-infra.yaml | 11 + config/endpoints.yaml | 29 - config/endpoints/ep-ansible-full.yaml | 6 + config/endpoints/ep-ansible-managed.yaml | 11 + config/endpoints/ep-ansible-oraclelinux.yaml | 8 + config/endpoints/ep-ansible-section9.yaml | 8 + config/enrichers.yaml | 2 - config/enrichers/example.yaml | 13 + config/projects.yaml | 13 - config/projects/prj-connectors-infra.yaml | 15 + config/sources.yaml | 75 - config/sources/src-granular.yaml | 10 + config/sources/src-infra.yaml | 8 + config/sources/src-section9-large.yaml | 12 + config/sources/src-section9.yaml | 17 + config/sources/src-ssh-example.yaml | 20 + config/{views.yaml => views/example.yaml} | 58 +- deploy/README.md | 10 +- deploy/k8s/deployment.yaml | 3 +- docker/Dockerfile | 2 +- docs/api.md | 54 +- docs/architecture.md | 3 +- docs/browsable-api.md | 13 +- docs/caching.md | 2 +- docs/cli.md | 65 +- docs/config-api.md | 115 +- docs/configuration.md | 820 +++++++--- docs/connectors.md | 333 +++- docs/deployment.md | 438 +++-- docs/endpoints.md | 39 +- docs/enrichers.md | 48 +- docs/federation.md | 136 +- docs/ldap-login.md | 20 +- docs/observability.md | 2 +- docs/on-demand-refresh.md | 195 ++- docs/projects.md | 29 +- docs/secrets.md | 6 +- docs/testing.md | 2 + docs/troubleshooting.md | 22 +- docs/views.md | 61 +- src/adapters/in/http/auth.rs | 16 +- src/adapters/in/http/config.rs | 54 +- src/adapters/in/http/connectors.rs | 212 +++ src/adapters/in/http/detail.rs | 9 +- src/adapters/in/http/html.rs | 4 +- src/adapters/in/http/index.rs | 4 + src/adapters/in/http/login.rs | 10 +- src/adapters/in/http/mod.rs | 1 + src/adapters/in/http/openapi.rs | 13 + src/adapters/in/http/projects.rs | 17 +- src/adapters/in/http/routes.rs | 25 +- src/adapters/in/http/session.rs | 46 +- src/adapters/in/http/settings.rs | 6 + src/adapters/in/http/sources.rs | 64 + src/adapters/in/http/spec.rs | 20 +- src/adapters/in/http/topology.rs | 132 +- src/adapters/in/http/ui_assets.rs | 62 +- src/adapters/in/http/views.rs | 29 + src/adapters/in/scheduler/mod.rs | 85 +- src/adapters/out/config/fs.rs | 148 +- src/adapters/out/git/cli.rs | 16 + src/adapters/out/peers/http.rs | 40 +- src/application/config.rs | 16 + src/application/enrich.rs | 24 +- src/application/host_search.rs | 2 +- src/application/projects.rs | 2 +- src/application/topology.rs | 500 +++++- src/config.rs | 948 ++++++++--- src/config/migrate.rs | 1507 ++++++++++++++++++ src/domain/api_key.rs | 3 + src/domain/connector.rs | 181 +++ src/domain/credential.rs | 3 + src/domain/endpoint.rs | 11 +- src/domain/enricher.rs | 12 +- src/domain/mod.rs | 1 + src/domain/project.rs | 7 +- src/domain/source.rs | 441 ++++- src/domain/topology.rs | 178 +++ src/domain/view.rs | 3 + src/domain/yaml.rs | 206 ++- src/lib.rs | 27 + src/main.rs | 116 +- src/ports/git.rs | 8 + src/ports/peer.rs | 13 +- src/state.rs | 29 +- tests/api_integration_test.rs | 44 +- tests/audit_log_test.rs | 3 +- tests/check_config_test.rs | 2 +- tests/config_api_test.rs | 90 +- tests/connectors_test.rs | 235 +++ tests/credentials_test.rs | 2 + tests/federation_test.rs | 171 ++ tests/fixtures/legacy-0.34/api_keys.yaml | 13 + tests/fixtures/legacy-0.34/config.yaml | 25 + tests/fixtures/legacy-0.34/credentials.yaml | 65 + tests/fixtures/legacy-0.34/endpoints.yaml | 13 + tests/fixtures/legacy-0.34/enrichers.yaml | 7 + tests/fixtures/legacy-0.34/projects.yaml | 13 + tests/fixtures/legacy-0.34/sources.yaml | 171 ++ tests/fixtures/legacy-0.34/views.yaml | 12 + tests/git_test.rs | 10 + tests/ldap_login_test.rs | 50 + tests/metrics_auth_reload_test.rs | 2 +- tests/migration_test.rs | 312 ++++ tests/on_demand_refresh_test.rs | 3 +- tests/remote_connector_test.rs | 9 +- tests/sync_test.rs | 13 +- tests/ui_assets_test.rs | 57 + tests/views_test.rs | 14 +- 122 files changed, 8128 insertions(+), 1408 deletions(-) delete mode 100644 config/api_keys.yaml create mode 100644 config/api_keys/example.yaml create mode 100644 config/connectors/granular.yaml create mode 100644 config/connectors/sample-infra.yaml create mode 100644 config/connectors/sample-inventory.yaml create mode 100644 config/connectors/ssh-facts.yaml delete mode 100644 config/credentials.yaml create mode 100644 config/credentials/cred-github-token.yaml create mode 100644 config/credentials/cred-section9-api.yaml create mode 100644 config/credentials/cred-ssh-infra.yaml delete mode 100644 config/endpoints.yaml create mode 100644 config/endpoints/ep-ansible-full.yaml create mode 100644 config/endpoints/ep-ansible-managed.yaml create mode 100644 config/endpoints/ep-ansible-oraclelinux.yaml create mode 100644 config/endpoints/ep-ansible-section9.yaml delete mode 100644 config/enrichers.yaml create mode 100644 config/enrichers/example.yaml delete mode 100644 config/projects.yaml create mode 100644 config/projects/prj-connectors-infra.yaml delete mode 100644 config/sources.yaml create mode 100644 config/sources/src-granular.yaml create mode 100644 config/sources/src-infra.yaml create mode 100644 config/sources/src-section9-large.yaml create mode 100644 config/sources/src-section9.yaml create mode 100644 config/sources/src-ssh-example.yaml rename config/{views.yaml => views/example.yaml} (54%) create mode 100644 src/adapters/in/http/connectors.rs create mode 100644 src/config/migrate.rs create mode 100644 src/domain/connector.rs create mode 100644 tests/connectors_test.rs create mode 100644 tests/federation_test.rs create mode 100644 tests/fixtures/legacy-0.34/api_keys.yaml create mode 100644 tests/fixtures/legacy-0.34/config.yaml create mode 100644 tests/fixtures/legacy-0.34/credentials.yaml create mode 100644 tests/fixtures/legacy-0.34/endpoints.yaml create mode 100644 tests/fixtures/legacy-0.34/enrichers.yaml create mode 100644 tests/fixtures/legacy-0.34/projects.yaml create mode 100644 tests/fixtures/legacy-0.34/sources.yaml create mode 100644 tests/fixtures/legacy-0.34/views.yaml create mode 100644 tests/migration_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ee9daab..fdccdb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,58 @@ project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- The browsable API's sync button reads "Sync" (was "Sync now"), as in the web UI. +- The image ships unified-ui 0.2.0 (`UI_REF`): the federation map on edges, a source's groups with their hosts and variables, exact live durations, and the shared look with unified-gitops. + +- **Breaking (everyone: every configuration directory):** the configuration is item files, the way AWX configuration as code is laid out. `config.yaml` stays at the root, unchanged; every other `*.yaml`/`*.yml`, at any depth and in any folder, holds items under a typed top-level key (`sources`, `views`, `endpoints`, `enrichers`, `connectors`, `projects`, `credentials`, `api_keys`). Each item has a `name` (its identity: what references, URLs and cache keys use — the 0.34 ids) and an optional `description` (its label). A name defined twice, an unknown top-level key or an unknown key inside an item fails the load naming the file and the item. The eight fixed files of id-keyed maps are gone; a 0.34 directory is converted at startup (see Upgrading below). See `docs/configuration.md`. +- **Breaking (anyone writing sources):** what a source said about its script moved to a new `connectors` item: `type` (`script` or `ssh`), `project`, `path` (the script inside the project — `project` + `path` always means a file in that checkout), `script_args`, `host_args`, `output_format` and a default `timeout_seconds`. A source names it with `connector:` and keeps only what is its own: credentials, `config`, schedule, TTLs and every per-source flag, plus an optional `timeout_seconds` override. Sources of the same system now share one connector instead of each carrying a copy. Two connectors are built in and need no item: `remote` (another unified-api: `config.url` and `config.source`, the source id on that instance — formerly `script_path`) and `static_inventory` (`project` + `path` on the source — formerly `project_id` + `script_path`). See `docs/connectors.md`. +- **Breaking (anyone writing references):** reference keys lost their `_id` suffix: a source's `credentials`, a project's `credential`, an enricher's `target`, `source` and `project`, an endpoint's `sources` and `project`. +- **Breaking (pipelines using the configuration API):** files are addressed by path, folders included: `GET/PUT/DELETE /api/v1/config/{path}` with slashes (`/api/v1/config/sources/acme/src-d42.yaml`). A whole-directory push with `prune: true` removes every file at any depth that it leaves out, and the folders it empties; `missing` only ever names `config.yaml`. A push still in the 0.34 layout is refused with a hint naming `--migrate-config`. +- **Breaking (`GET /api/v1/sources/{id}` admin callers):** `gathering.project_id` is null for remote and ssh sources, which run nothing from a checkout (it used to repeat a project that was required and never used). + +### Added + +- Edges show the whole federation. Every minute, an instance with remote sources pushes its map to each peer (`PUT /api/v1/topology/federation`, with the key its remote sources already use); admins read it at `GET /api/v1/topology/federation`. Nothing to configure on the edges. The map an edge receives is redacted the way a restricted caller's is: its peers' names and versions, not their own maps, and no error that names the central's secrets. Only a key sent with the request may push, never a browser session. See `docs/federation.md`. +- `GET /api/v1/projects` answers each project's `revision` (the commit its checkout is at) and `credential_id`, so the UI's project cards show the commit and the credential, as unified-gitops does. +- `GET /api/v1/topology`: each peer source's `last_pull_age_seconds`, when data last moved along that link (its last successful sync); the UI animates a link each time it does. +- `GET /api/v1/sources/{id}/groups/{group}`: one group's variables, child groups and host count, for sources and views, without fetching the whole dataset. +- `server.name` (optional, reloadable): what the instance is called on the Topology page; in its own topology answer and in the map it pushes. Without it the UI derives one from the URL. +- `server.url` (optional, reloadable): this instance's public address. A central sends it with the federation map so edges can link back to it; also shown on `GET /api/v1/settings`. + +- A browser that is not signed in is sent to the login form by the server when it asks for a page of the UI, so the app never loads only to redirect itself (no flash of the overview before the login page). + +- `GET /api/v1/connectors` and `GET /api/v1/connectors/{id}` (admin-only): each connector, the two built-ins included (`built_in: true`), what it runs, the sources that use it with the timeout each syncs under, and a health summary of those sources. Linked from the API index as `connectors`. +- `gathering.connector` on a source's page, `used_by.connectors` on projects, and `connectors` in the configuration API's summary and reload report. +- `unified-api --migrate-config [dir]`: converts a 0.34 directory in place, for a configuration repository to run once and commit. + +### Fixed + +- A browser session opened with an API key is ended by a reload that removes that key's `name` (its identity), not by its label. Two keys with the same label, one of them removed, left the removed key's sessions working with its old permissions until they expired. + +### Upgrading from 0.34 + +Response shapes and ids are unchanged: consumers see no difference. On +startup, a configuration directory still in the 0.34 layout is converted once, +before it is loaded: connectors are extracted from the sources and +de-duplicated, references renamed, one file written per item +(`
/.yaml`), and the old files moved to a hidden +`.legacy-config-/` folder; the log says what was done. Values are +written as they were (`1.10`, `0x1F` and `True` stay exactly that), and an +unquoted value in a text field is still text (`script_args: ["--days", 7]`). +A generated connector never takes a name already in use, whether another +connector's or a built-in's (a script in a `remote/` folder becomes `remote-2`). Nothing is +moved unless the converted configuration loads — otherwise the process refuses +to start with the directory untouched. A read-only directory (a ConfigMap) is +converted into a temporary copy instead, every start; the copy is removed +when the process stops, or by the next start after a crash. Comments in the old +files are not carried over (the backup keeps them). Convert the source of the +files too — `unified-api --migrate-config `, or by hand — or the next +push puts the old layout back; `--check-config` on a 0.34 directory checks the +conversion without writing. Rolling back to 0.34 needs the old files restored +from the backup. + ## [0.34.0] - 2026-10-25 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 6213431..885ce2f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,10 +42,13 @@ src/ ├── main.rs # Entrypoint: load config, build app, start Axum ├── lib.rs # Module tree + AppBuilder (composition root) ├── state.rs # AppState (ports as Arc + static config) -├── config.rs # YAML configuration loading from config/ directory +├── config.rs # Loader: config.yaml + item files at any depth (SECTIONS) +├── config/ +│ └── migrate.rs # one-time 0.34 → item-file conversion (startup, --migrate-config) ├── domain/ # Core domain types (pure, no dependencies) │ ├── dataset.rs # Dataset, Group, HostVars -│ ├── source.rs # Source, TtlOverrides, ConnectorType +│ ├── source.rs # SourceDef (file) → Source (resolved with its connector), TtlOverrides +│ ├── connector.rs # Connector item (how to talk to a system); built-ins remote/static_inventory │ ├── cache_entry.rs # CacheEntry with TTL logic │ ├── sync_health.rs # SyncHealth + registry (one per kind: sync/enrich/project/snapshot) │ ├── credential.rs # Credential, CredentialType @@ -95,6 +98,7 @@ src/ │ │ │ ├── hosts.rs # PUT/DELETE host │ │ │ ├── endpoints.rs # Output endpoints (GET and POST) │ │ │ ├── projects.rs # Git project routes (admin-only) +│ │ │ ├── connectors.rs # GET connectors + their users and health (admin-only) │ │ │ ├── health.rs # /healthz, /readyz │ │ │ ├── metrics.rs # /metrics (Prometheus exporter, installed once) │ │ │ ├── login.rs # login page, sessions, logout @@ -115,8 +119,9 @@ src/ │ └── secrets/ # env.rs: SecretsPort → env/JSON files; mock.rs: test double; │ # vault.rs; cache.rs; reloadable.rs (swappable chain); │ # build_chain() — the one place the chain is wired -config/ # Split YAML config (server, credentials, sources, etc.) +config/ # Example config: config.yaml + item files (sources/, connectors/, ...) tests/ # Integration tests (*.rs), with sample scripts mirroring src/adapters/out/: +├── fixtures/legacy-0.34/ # a 0.34-layout directory the migration tests convert └── adapters/ └── out/ # sample scripts — stand-ins for the driven adapters ├── connectors/ # sample source connectors (incl. slow.py for timeout tests) @@ -136,6 +141,19 @@ translators that call it; don't put orchestration logic in either. No external data dependencies (no Redis, no PostgreSQL). All cache in-memory with DashMap. Cache mutations must use the atomic `CachePort::update`/`merge_or_insert` operations — never the get → modify → set pattern (it loses concurrent writes). +Configuration is item files (AWX style): `config.yaml` at the root, every other +`*.yaml`/`*.yml` at any depth holds items under a typed top-level key +(`config::SECTIONS`); `name` is the identity (what references, URLs and cache keys +use), `description` the label. Struct `name` fields hold that LABEL — `config::take` +moves `description` into them, which is what keeps API `name` fields unchanged. +References are written without `_id` (`credentials`, `project`, `target`, `sources`); +Rust fields kept their old names via `#[serde(rename)]`, because the API still +answers `target_id` etc. A source item (`SourceDef`) is resolved with its connector +at load into the flat runtime `Source` that everything downstream reads — never +read a connector at sync time. A new item type touches `SECTIONS`, the loader's +`take` calls, `AppConfig`, and `config::migrate`'s key order. A 0.34 directory is +converted at startup (`config::migrate`): staged, validated, backed up to +`.legacy-config-*`, never half-written; a read-only one into a temp copy. Configuration from YAML files, parsed STRICTLY: every config struct carries `deny_unknown_fields`, so a typo'd key fails startup naming the key instead of silently applying the default (policy comment in `config.rs`). Free-form data @@ -222,7 +240,7 @@ Vault (KV v2, token or Kubernetes auth) per credential via `vault_path` + removal) and one naming an api-key env var that is not set. See `docs/config-api.md`. -- **Views:** `views.yaml` declares read-only composites over several sources. +- **Views:** `views` items declare read-only composites over several sources. A view is served on the SOURCE routes and shares their id space (config validation rejects a collision) — that is what makes migrating a consumer a one-word change. It holds no cache entry: `application::views::snapshot` @@ -268,7 +286,7 @@ Vault (KV v2, token or Kubernetes auth) per credential via `vault_path` + waits for every configured source. Either way the body carries `sources_synced` and `sources_pending`, so a failing probe names what it is waiting for. -- **Auth:** keys are declared in `api_keys.yaml`, each `role: admin` (everything) +- **Auth:** keys are declared as `api_keys` items, each `role: admin` (everything) or restricted to explicit `sources`/`endpoints` id lists; secrets come from the env var each definition names. The legacy `UNIFIED_API_KEY` still works as one extra admin key. Constant-time compare; `/healthz`, `/readyz` and Swagger diff --git a/README.md b/README.md index c9207a6..a8e55c7 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ and every one of them sees the same inventory. ## ✨ Features - **Pluggable sources**: any executable that prints inventory JSON is a connector +- **Connectors**: one item says how to read a system; every instance of it is a source naming it - **SSH connector**: gathers Ansible facts from whole fleets in parallel - **In-memory cache with TTLs**: per-dataset, per-host and per-group freshness, no database - **Gzip responses**: a client that accepts gzip transfers about a tenth of the bytes @@ -28,6 +29,7 @@ and every one of them sees the same inventory. - **Views**: one id over several sources, routing each host to the member that owns it - **Scheduled + on-demand sync**: interval sync per source, plus scoped sync over the API - **Refresh**: a read can bring the hosts it names up to date, bounded by the source's TTL +- **Config as code**: AWX-style item files, one per source, in any folders you like - **Configuration API**: a pipeline pushes the config directory and reloads it live, no restart - **Web UI**: sources, endpoints, topology, metrics and host search in a browser, same address - **Swagger UI**: interactive OpenAPI docs served at `/swagger-ui/` @@ -41,13 +43,15 @@ and every one of them sees the same inventory. |---|---| | [API](docs/api.md) | All routes with authentication, status code semantics and curl examples | | [Architecture](docs/architecture.md) | The four layers, the dependency rule and the concurrency model | -| [CLI](docs/cli.md) | Environment variables, log tuning, health checks and common curl operations | -| [Configuration](docs/configuration.md) | Every YAML file field by field, env vars and startup validation | +| [Browsable API](docs/browsable-api.md) | The API's own pages in a browser: login, sessions, navigation | +| [CLI](docs/cli.md) | `--check-config`, `--migrate-config`, env vars, log tuning and common curl operations | +| [Configuration](docs/configuration.md) | The item-file layout, every item field by field, upgrading from 0.34 | | [Configuration API](docs/config-api.md) | Pushing the config directory over HTTP, validating it and reloading it live | -| [Connectors](docs/connectors.md) | Script contracts for connectors, enrichers and output transformers | +| [Connectors](docs/connectors.md) | Connector items, the built-ins, and the script contracts for all three kinds | | [Deployment](docs/deployment.md) | Container image, worked config example, CI/CD, Kubernetes and ArgoCD | | [Enrichers](docs/enrichers.md) | Post-processing cached data: modes, triggers, freshness rules, health | | [Federation](docs/federation.md) | One instance per datacenter, one central federating them, no WAN SSH | +| [LDAP login](docs/ldap-login.md) | Signing in with a directory account, and the access rules that decide what it reaches | | [Observability](docs/observability.md) | Scheduling, structured logs and the metrics worth alerting on | | [Output endpoints](docs/endpoints.md) | Consumer-facing transformers: contract, GET vs POST, permissions | | [Projects](docs/projects.md) | Git checkouts: sync styles, script resolution, virtualenvs, health | diff --git a/config/api_keys.yaml b/config/api_keys.yaml deleted file mode 100644 index 6d025d5..0000000 --- a/config/api_keys.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# API keys with per-consumer permissions. The secret is never in this file: -# `env` names the environment variable that holds it. No keys defined and no -# UNIFIED_API_KEY = open API (a warning is logged at startup). -# -# key-awx: -# name: "AWX" -# env: "UNIFIED_API_KEY_AWX" -# role: "admin" # everything -# -# key-forms: -# name: "AnsibleForms" -# env: "UNIFIED_API_KEY_FORMS" -# # role defaults to "restricted": only what is listed below -# sources: ["src-section9"] # may list/read/sync these sources -# endpoints: ["ep-ansible-full"] # may list/run these endpoints diff --git a/config/api_keys/example.yaml b/config/api_keys/example.yaml new file mode 100644 index 0000000..aaa2f93 --- /dev/null +++ b/config/api_keys/example.yaml @@ -0,0 +1,17 @@ +--- +# API keys with per-consumer permissions. The secret is never in this file: +# `env` names the environment variable that holds it. No keys defined and no +# UNIFIED_API_KEY = open API (a warning is logged at startup). +# +# api_keys: +# - name: key-awx +# description: "AWX" +# env: "UNIFIED_API_KEY_AWX" +# role: "admin" # everything +# +# - name: key-forms +# description: "AnsibleForms" +# env: "UNIFIED_API_KEY_FORMS" +# # role defaults to "restricted": only what is listed below +# sources: ["src-section9"] # may list/read/sync these sources +# endpoints: ["ep-ansible-full"] # may list/run these endpoints diff --git a/config/config.yaml b/config/config.yaml index a49c4c3..a58628b 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,7 +3,7 @@ server: port: 8182 # The web UI on this same address: the image ships a build of unified-ui at # /usr/share/unified-api/ui. Sign-in is the API's own login page (an API key, - # or LDAP: docs/ldap-login.md); with no api_keys.yaml the API is open and so + # or LDAP: docs/ldap-login.md); with no api_keys items the API is open and so # is the UI. Remove the block for an API without a UI. See docs/ui.md. ui: enabled: true @@ -19,7 +19,7 @@ server: # Optional: let an admin key read, write and reload this directory over HTTP, # so a configuration pipeline can push a change instead of publishing an # artifact this instance has to pull. OFF by default — turning it on means an -# admin key can rewrite every file the loader reads, api_keys.yaml included. +# admin key can rewrite every file the loader reads, api_keys included. # See docs/config-api.md. # config_api: # enabled: true diff --git a/config/connectors/granular.yaml b/config/connectors/granular.yaml new file mode 100644 index 0000000..23e874c --- /dev/null +++ b/config/connectors/granular.yaml @@ -0,0 +1,16 @@ +--- +# A script that can gather ONE host without gathering the inventory. The full +# sync calls it with script_args, a `?host=` sync with host_args instead: +# +# POST /api/v1/sources/src-granular/sync -> --list +# POST /api/v1/sources/src-granular/sync?host=motoko.section9.net +# -> --only-host motoko.section9.net +# +# Try it and read `called_with` on the host to see which mode ran. +connectors: + - name: granular + description: "Granular inventory script" + project: "prj-connectors-infra" + path: "tests/adapters/out/connectors/granular.py" + script_args: ["--list"] + host_args: ["--only-host", "{target}"] diff --git a/config/connectors/sample-infra.yaml b/config/connectors/sample-infra.yaml new file mode 100644 index 0000000..d7aec40 --- /dev/null +++ b/config/connectors/sample-infra.yaml @@ -0,0 +1,6 @@ +--- +connectors: + - name: sample-infra + description: "Sample infrastructure data" + project: "prj-connectors-infra" + path: "tests/adapters/out/connectors/infra.py" diff --git a/config/connectors/sample-inventory.yaml b/config/connectors/sample-inventory.yaml new file mode 100644 index 0000000..2d156f3 --- /dev/null +++ b/config/connectors/sample-inventory.yaml @@ -0,0 +1,14 @@ +--- +# A connector says HOW to read a kind of system: which script, in which +# project, with which arguments. Every source of that system names it +# (`connector: sample-inventory`) and adds only what is its own: credentials, +# config, schedule, TTLs. See docs/connectors.md. +connectors: + - name: sample-inventory + description: "Sample inventory script" + type: "script" # script (default) | ssh + project: "prj-connectors-infra" # the checkout `path` resolves in + path: "tests/adapters/out/connectors/inventory.py" + # script_args: ["--list"] # for an Ansible dynamic inventory script + # output_format: "ansible" # native (default) | ansible + # timeout_seconds: 300 # default; a source may override it diff --git a/config/connectors/ssh-facts.yaml b/config/connectors/ssh-facts.yaml new file mode 100644 index 0000000..d783a6e --- /dev/null +++ b/config/connectors/ssh-facts.yaml @@ -0,0 +1,10 @@ +--- +# SSH: the native connector, nothing installed on the hosts. `path` is what +# runs remotely — a label such as "gather_facts" for the built-in gather +# modes, or the command itself for `gather_mode: script`. No project: nothing +# runs from a checkout. +connectors: + - name: ssh-facts + description: "SSH fact gathering" + type: "ssh" + path: "gather_facts" diff --git a/config/credentials.yaml b/config/credentials.yaml deleted file mode 100644 index 50c87d6..0000000 --- a/config/credentials.yaml +++ /dev/null @@ -1,24 +0,0 @@ -cred-section9-api: - name: "Section 9 API" - type: "username_password" - env_prefix: "SECTION9" - secret_keys: - username: "USERNAME" - password: "PASSWORD" - -cred-github-token: - name: "GitHub Token" - type: "token" - env_prefix: "GITHUB" - secret_keys: - token: "TOKEN" - -cred-ssh-infra: - name: "SSH Infrastructure" - type: "ssh_key" - env_prefix: "INFRA_SSH" - secret_keys: - username: "USERNAME" - file_keys: - ssh_key: "/run/secrets/infra-ssh/id_rsa" - ca_cert: "/run/secrets/infra-ssh/ca.pem" diff --git a/config/credentials/cred-github-token.yaml b/config/credentials/cred-github-token.yaml new file mode 100644 index 0000000..7087c2e --- /dev/null +++ b/config/credentials/cred-github-token.yaml @@ -0,0 +1,8 @@ +--- +credentials: + - name: cred-github-token + description: "GitHub Token" + type: "token" + env_prefix: "GITHUB" + secret_keys: + token: "TOKEN" diff --git a/config/credentials/cred-section9-api.yaml b/config/credentials/cred-section9-api.yaml new file mode 100644 index 0000000..2d90665 --- /dev/null +++ b/config/credentials/cred-section9-api.yaml @@ -0,0 +1,11 @@ +--- +# A credential names where its secrets are read from — never the secrets. +# This one reads SECTION9_USERNAME and SECTION9_PASSWORD. +credentials: + - name: cred-section9-api + description: "Section 9 API" + type: "username_password" + env_prefix: "SECTION9" + secret_keys: + username: "USERNAME" + password: "PASSWORD" diff --git a/config/credentials/cred-ssh-infra.yaml b/config/credentials/cred-ssh-infra.yaml new file mode 100644 index 0000000..22d2b6d --- /dev/null +++ b/config/credentials/cred-ssh-infra.yaml @@ -0,0 +1,11 @@ +--- +credentials: + - name: cred-ssh-infra + description: "SSH Infrastructure" + type: "ssh_key" + env_prefix: "INFRA_SSH" + secret_keys: + username: "USERNAME" + file_keys: + ssh_key: "/run/secrets/infra-ssh/id_rsa" + ca_cert: "/run/secrets/infra-ssh/ca.pem" diff --git a/config/endpoints.yaml b/config/endpoints.yaml deleted file mode 100644 index c95cd66..0000000 --- a/config/endpoints.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -ep-ansible-full: - name: "Full Ansible Inventory" - source_ids: ["src-section9", "src-infra"] - output: ansible - -ep-ansible-section9: - name: "Section 9 Only" - source_ids: ["src-section9", "src-infra"] - output: ansible - config: - filter: "datacenter=section9" - -ep-ansible-oraclelinux: - name: "OracleLinux Hosts" - source_ids: ["src-section9", "src-infra"] - output: ansible - config: - filter: "os=OracleLinux" - -# A constructed inventory: merge both sources, but return only the hosts the -# inventory of record has. src-infra still contributes its variables and its -# groups for those hosts — it just cannot add hosts of its own. -ep-ansible-managed: - name: "Managed Hosts Only" - source_ids: ["src-section9", "src-infra"] - output: ansible - limit: - by_hosts_from_inventory: "src-section9" diff --git a/config/endpoints/ep-ansible-full.yaml b/config/endpoints/ep-ansible-full.yaml new file mode 100644 index 0000000..1eedbd9 --- /dev/null +++ b/config/endpoints/ep-ansible-full.yaml @@ -0,0 +1,6 @@ +--- +endpoints: + - name: ep-ansible-full + description: "Full Ansible Inventory" + sources: ["src-section9", "src-infra"] + output: "ansible" diff --git a/config/endpoints/ep-ansible-managed.yaml b/config/endpoints/ep-ansible-managed.yaml new file mode 100644 index 0000000..b9d4823 --- /dev/null +++ b/config/endpoints/ep-ansible-managed.yaml @@ -0,0 +1,11 @@ +--- +# A constructed inventory: merge both sources, but return only the hosts the +# inventory of record has. src-infra still contributes its variables and its +# groups for those hosts — it just cannot add hosts of its own. +endpoints: + - name: ep-ansible-managed + description: "Managed Hosts Only" + sources: ["src-section9", "src-infra"] + output: "ansible" + limit: + by_hosts_from_inventory: "src-section9" diff --git a/config/endpoints/ep-ansible-oraclelinux.yaml b/config/endpoints/ep-ansible-oraclelinux.yaml new file mode 100644 index 0000000..7b42369 --- /dev/null +++ b/config/endpoints/ep-ansible-oraclelinux.yaml @@ -0,0 +1,8 @@ +--- +endpoints: + - name: ep-ansible-oraclelinux + description: "OracleLinux Hosts" + sources: ["src-section9", "src-infra"] + output: "ansible" + config: + filter: "os=OracleLinux" diff --git a/config/endpoints/ep-ansible-section9.yaml b/config/endpoints/ep-ansible-section9.yaml new file mode 100644 index 0000000..d8f9650 --- /dev/null +++ b/config/endpoints/ep-ansible-section9.yaml @@ -0,0 +1,8 @@ +--- +endpoints: + - name: ep-ansible-section9 + description: "Section 9 Only" + sources: ["src-section9", "src-infra"] + output: "ansible" + config: + filter: "datacenter=section9" diff --git a/config/enrichers.yaml b/config/enrichers.yaml deleted file mode 100644 index 0b61c53..0000000 --- a/config/enrichers.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# Enrichers — post-processors that enrich already-cached data. -# They receive the current dataset on stdin and return the modified hosts on stdout. diff --git a/config/enrichers/example.yaml b/config/enrichers/example.yaml new file mode 100644 index 0000000..94fd95b --- /dev/null +++ b/config/enrichers/example.yaml @@ -0,0 +1,13 @@ +--- +# Enrichers — post-processors that enrich already-cached data. +# They receive the current dataset on stdin and return the modified hosts on +# stdout (script mode), or copy named fields from one source onto another +# (declarative mode). See docs/enrichers.md. +# +# enrichers: +# - name: enr-infra-onto-section9 +# description: "Infrastructure fields onto the Section 9 hosts" +# target: "src-section9" # the cache entry written into +# source: "src-infra" # declarative: copy from this source... +# fields: ["rack", "power"] # ...these vars +# sync_interval_seconds: 300 diff --git a/config/projects.yaml b/config/projects.yaml deleted file mode 100644 index 9775974..0000000 --- a/config/projects.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# The sample sources' scripts live in this repository (tests/adapters/out/), -# so the sample project is unified-api itself: public, so no credential, and -# the paths in sources.yaml resolve inside its checkout. A real deployment -# points this at its own connector repository. -prj-connectors-infra: - name: "Infrastructure Connectors" - git_url: "https://github.com/OpusProjects/unified-api.git" - # branch: "main" # default "main" - # credential_id: "cred-github-token" # for private repos (token or ssh_key) - sync_interval_seconds: 1800 # re-pull every 30 min (0/absent = no periodic sync) - # schedule: "15 2 * * *" # cron (UTC) instead of the interval — pick one - # sync_on_boot: false # start from the existing checkout as-is; - # # update only via POST /api/v1/projects/{id}/sync diff --git a/config/projects/prj-connectors-infra.yaml b/config/projects/prj-connectors-infra.yaml new file mode 100644 index 0000000..e037f8e --- /dev/null +++ b/config/projects/prj-connectors-infra.yaml @@ -0,0 +1,15 @@ +--- +# The sample connectors' scripts live in this repository (tests/adapters/out/), +# so the sample project is unified-api itself: public, so no credential, and +# the connectors' paths resolve inside its checkout. A real deployment points +# this at its own connector repository. +projects: + - name: prj-connectors-infra + description: "Infrastructure Connectors" + git_url: "https://github.com/OpusProjects/unified-api.git" + # branch: "main" # default "main" + # credential: "cred-github-token" # for private repos (token or ssh_key) + sync_interval_seconds: 1800 # re-pull every 30 min (0/absent = no periodic sync) + # schedule: "15 2 * * *" # cron (UTC) instead of the interval — pick one + # sync_on_boot: false # start from the existing checkout as-is; + # # update only via POST /api/v1/projects/{id}/sync diff --git a/config/sources.yaml b/config/sources.yaml deleted file mode 100644 index 2e50e75..0000000 --- a/config/sources.yaml +++ /dev/null @@ -1,75 +0,0 @@ -src-section9: - name: "Section 9 Inventory" - project_id: "prj-connectors-infra" - script_path: "tests/adapters/out/connectors/inventory.py" - credential_ids: ["cred-section9-api"] - # schedule: "0 */1 * * *" # cron (UTC) instead of the interval — pick one - sync_interval_seconds: 60 - ttl_seconds: 3600 - ttl_overrides: - groups: - production: 900 - hosts: - motoko.section9.net: 300 - -src-infra: - name: "Infrastructure Data" - project_id: "prj-connectors-infra" - script_path: "tests/adapters/out/connectors/infra.py" - credential_ids: [] - sync_interval_seconds: 300 - ttl_seconds: 3600 - sync_mode: "replace" - -src-section9-large: - name: "Section 9 Large Inventory" - project_id: "prj-connectors-infra" - script_path: "tests/adapters/out/connectors/inventory.py" - credential_ids: [] - # Cron cadence (UTC): every 2 hours on the hour, instead of an interval - schedule: "0 */2 * * *" - ttl_seconds: 7200 - config: - scenario: "large" - -src-ssh-example: - name: "SSH Facts Gathering" - connector_type: "ssh" - project_id: "prj-connectors-infra" - script_path: "gather_facts" - credential_ids: ["cred-ssh-infra"] - sync_interval_seconds: 300 - ttl_seconds: 600 - config: - hosts: "host1.example.com,host2.example.com" - port: "22" - concurrency: "50" - ssh_connect_timeout_seconds: "30" - gather_mode: "facts" - fact_path: "/etc/ansible/facts.d" - # Host key verification: point at ssh-keyscan output to refuse unknown or - # changed server keys BEFORE authenticating (docs/connectors.md). Left - # commented because startup validation requires the file to exist; - # without it any key is accepted and every sync logs a warning. - # ssh_known_hosts: "/etc/unified-api/known_hosts" - -# A script that can gather ONE host without gathering the inventory. The full -# sync calls it with script_args, a `?host=` sync with host_args instead: -# -# POST /api/v1/sources/src-granular/sync -> --list -# POST /api/v1/sources/src-granular/sync?host=motoko.section9.net -# -> --only-host motoko.section9.net -# -# Try it and read `called_with` on the host to see which mode ran. -src-granular: - name: "Granular Inventory" - project_id: "prj-connectors-infra" - script_path: "tests/adapters/out/connectors/granular.py" - script_args: ["--list"] - host_args: ["--only-host", "{target}"] - # The granular mode describes the groups it puts the host in, so take their - # vars as well (off by default) - host_sync_updates_group_vars: true - credential_ids: [] - sync_interval_seconds: 300 - ttl_seconds: 3600 diff --git a/config/sources/src-granular.yaml b/config/sources/src-granular.yaml new file mode 100644 index 0000000..213f3e3 --- /dev/null +++ b/config/sources/src-granular.yaml @@ -0,0 +1,10 @@ +--- +sources: + - name: src-granular + description: "Granular Inventory" + connector: "granular" + # The granular mode describes the groups it puts the host in, so take their + # vars as well (off by default) + host_sync_updates_group_vars: true + sync_interval_seconds: 300 + ttl_seconds: 3600 diff --git a/config/sources/src-infra.yaml b/config/sources/src-infra.yaml new file mode 100644 index 0000000..2c833f3 --- /dev/null +++ b/config/sources/src-infra.yaml @@ -0,0 +1,8 @@ +--- +sources: + - name: src-infra + description: "Infrastructure Data" + connector: "sample-infra" + sync_interval_seconds: 300 + ttl_seconds: 3600 + sync_mode: "replace" diff --git a/config/sources/src-section9-large.yaml b/config/sources/src-section9-large.yaml new file mode 100644 index 0000000..91fa2fa --- /dev/null +++ b/config/sources/src-section9-large.yaml @@ -0,0 +1,12 @@ +--- +# The same connector as src-section9, another instance of it: only the config +# (and the cadence) differ. +sources: + - name: src-section9-large + description: "Section 9 Large Inventory" + connector: "sample-inventory" + # Cron cadence (UTC): every 2 hours on the hour, instead of an interval + schedule: "0 */2 * * *" + ttl_seconds: 7200 + config: + scenario: "large" diff --git a/config/sources/src-section9.yaml b/config/sources/src-section9.yaml new file mode 100644 index 0000000..ac3dba6 --- /dev/null +++ b/config/sources/src-section9.yaml @@ -0,0 +1,17 @@ +--- +# A source says WHICH instance of a system: its connector, its credentials, its +# config, when to sync and how long the data stays fresh. `name` is its +# identity — the id in every URL and reference; `description` is the label. +sources: + - name: src-section9 + description: "Section 9 Inventory" + connector: "sample-inventory" + credentials: ["cred-section9-api"] + # schedule: "0 */1 * * *" # cron (UTC) instead of the interval — pick one + sync_interval_seconds: 60 + ttl_seconds: 3600 + ttl_overrides: + groups: + production: 900 + hosts: + motoko.section9.net: 300 diff --git a/config/sources/src-ssh-example.yaml b/config/sources/src-ssh-example.yaml new file mode 100644 index 0000000..5e80b59 --- /dev/null +++ b/config/sources/src-ssh-example.yaml @@ -0,0 +1,20 @@ +--- +sources: + - name: src-ssh-example + description: "SSH Facts Gathering" + connector: "ssh-facts" + credentials: ["cred-ssh-infra"] + sync_interval_seconds: 300 + ttl_seconds: 600 + config: + hosts: "host1.example.com,host2.example.com" + port: "22" + concurrency: "50" + ssh_connect_timeout_seconds: "30" + gather_mode: "facts" + fact_path: "/etc/ansible/facts.d" + # Host key verification: point at ssh-keyscan output to refuse unknown or + # changed server keys BEFORE authenticating (docs/connectors.md). Left + # commented because startup validation requires the file to exist; + # without it any key is accepted and every sync logs a warning. + # ssh_known_hosts: "/etc/unified-api/known_hosts" diff --git a/config/views.yaml b/config/views/example.yaml similarity index 54% rename from config/views.yaml rename to config/views/example.yaml index fd9ce1d..c94c270 100644 --- a/config/views.yaml +++ b/config/views/example.yaml @@ -1,3 +1,4 @@ +--- # Views: read-only composites over several sources. # # A view presents its members as ONE id, routes a per-host read to whichever @@ -9,40 +10,41 @@ # a one-word change to an id. The write routes (POST /sync, DELETE, host # PUT/DELETE) refuse a view id and say why. # -# Why it exists: federation (connector_type: remote) means a central needs no +# Why it exists: federation (connector: remote) means a central needs no # credentials into a datacenter, because the edge that owns the hosts does the # gathering. But the CONSUMER still had to know the topology, since the facts of # a datacenter B host lived under a different source id than those of a # datacenter A host — and every new edge made every consumer relearn the split. # -# vw-facts-all: -# name: "Facts, both datacenters" +# views: +# - name: vw-facts-all +# description: "Facts, both datacenters" # -# # Order matters: the FIRST member that claims a host wins it. Overlap is -# # normal (a multi-homed host legitimately sits in two datacenter groups) and -# # needs a rule that is stable rather than clever. -# members: -# - source: "src-ssh-dc1" # where the facts come from -# owns: -# source: "src-d42" # inventory the pattern resolves against -# groups: ["datacenter_dc1"] # this member owns those groups' members -# - source: "src-edge-dc2" # a remote (federated) member works too -# owns: -# source: "src-d42" -# groups: ["datacenter_dc2"] -# # hosts: ["appliance01.dc2.example"] # claimed literally, whether or -# # # not the inventory knows them +# # Order matters: the FIRST member that claims a host wins it. Overlap is +# # normal (a multi-homed host legitimately sits in two datacenter groups) and +# # needs a rule that is stable rather than clever. +# members: +# - source: "src-ssh-dc1" # where the facts come from +# owns: +# source: "src-d42" # inventory the pattern resolves against +# groups: ["datacenter_dc1"] # this member owns those groups' members +# - source: "src-edge-dc2" # a remote (federated) member works too +# owns: +# source: "src-d42" +# groups: ["datacenter_dc2"] +# # hosts: ["appliance01.dc2.example"] # claimed literally, whether or +# # # not the inventory knows them # -# # Optional. Absent = each host inherits its owning member's TTL, so the -# # member keeps governing. Set it and the view has a freshness policy of its -# # own — which is also the GATE for on-demand refresh, since a read only -# # gathers hosts older than the TTL. Two doors onto the same data with -# # different refresh rules is fine as a deliberate consumer-facing policy; -# # what it must never be is an accident. -# # -# # A member's per-host and per-group ttl_overrides still win over this: an -# # override beats the default, here as everywhere else. -# # ttl_seconds: 30 +# # Optional. Absent = each host inherits its owning member's TTL, so the +# # member keeps governing. Set it and the view has a freshness policy of its +# # own — which is also the GATE for on-demand refresh, since a read only +# # gathers hosts older than the TTL. Two doors onto the same data with +# # different refresh rules is fine as a deliberate consumer-facing policy; +# # what it must never be is an accident. +# # +# # A member's per-host and per-group ttl_overrides still win over this: an +# # override beats the default, here as everywhere else. +# # ttl_seconds: 30 # # Ownership is DECLARED, not inferred from what a member happens to have cached. # Two reasons, both found the hard way: @@ -64,5 +66,5 @@ # nobody investigates. # # API keys grant a view exactly like a source, by its id under `sources:` in -# api_keys.yaml. A key granted the view needs no access to the members — the +# an api_keys item. A key granted the view needs no access to the members — the # view is the contract, the members are internal topology. diff --git a/deploy/README.md b/deploy/README.md index a155779..c59c873 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -11,8 +11,16 @@ apply files instead of copy-pasting from prose, and so CI can validate them PVC; `kubectl apply -k deploy/k8s/`. You supply the named ConfigMap and Secrets: +A ConfigMap is flat (`--from-file` does not recurse, and a key cannot hold a `/`), while the +configuration is item files in folders (`sources/`, `connectors/`...). The loader does not care +where a file sits — the top-level key says what it holds — so flatten the tree into the keys, as +below, or give each key a `path:` under `items:` in the volume to rebuild the folders. A +directory that changes often is better pushed with the configuration API onto a PVC. + ```bash -kubectl create configmap unified-api-config --from-file=config/ +# config/sources/src-a.yaml becomes the key sources-src-a.yaml +kubectl create configmap unified-api-config $(cd config && find . -name '*.yaml' \ + | sed 's|^\./||' | while read -r f; do printf -- '--from-file=%s=config/%s ' "${f//\//-}" "$f"; done) kubectl create secret generic unified-api-env --from-env-file=.env kubectl create secret generic unified-api-secrets \ --from-file=ssh-private-key=secrets/id_ed25519 \ diff --git a/deploy/k8s/deployment.yaml b/deploy/k8s/deployment.yaml index cc46969..d2a1261 100644 --- a/deploy/k8s/deployment.yaml +++ b/deploy/k8s/deployment.yaml @@ -3,7 +3,8 @@ # into invalid the way YAML in prose can. # # What the names refer to: -# unified-api-config ConfigMap holding the config/*.yaml files (CONFIG_DIR) +# unified-api-config ConfigMap holding config.yaml and the item files (CONFIG_DIR; +# flat, see deploy/README.md for folders) # unified-api-env Secret with the env table (API keys, credential values) # unified-api-secrets Secret with file-delivered secrets (SSH key, JSON files) # unified-api-state PVC for project checkouts and the cache snapshot diff --git a/docker/Dockerfile b/docker/Dockerfile index abcaaa8..d4e30b0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,7 @@ # versions; each unified-api release pins the one it ships (UI_REF), so an # image always serves the UI it was tested with. FROM node:22-alpine AS ui -ARG UI_REF=v0.1.0 +ARG UI_REF=v0.2.0 RUN apk add --no-cache git \ && git clone --depth 1 --branch "$UI_REF" https://github.com/OpusProjects/unified-ui.git /ui WORKDIR /ui diff --git a/docs/api.md b/docs/api.md index c007959..b10cf26 100644 --- a/docs/api.md +++ b/docs/api.md @@ -10,13 +10,15 @@ reference; the spec is generated from the code and is always authoritative. - [Sources](#sources) - [Enrichers](#enrichers) - [Output endpoints](#output-endpoints) +- [Connectors (admin-only)](#connectors-admin-only) - [Projects (admin-only)](#projects-admin-only) +- [Configuration (admin-only)](#configuration-admin-only) --- ## Authentication -API keys are defined in `api_keys.yaml` (see [configuration](configuration.md)); +API keys are defined as `api_keys` items (see [configuration](configuration.md#api_keys)); each key's secret lives in the environment variable the definition names, never in the YAML. The legacy `UNIFIED_API_KEY` environment variable still works and acts as one extra admin key. Either header authenticates: @@ -75,8 +77,8 @@ dashboard's origin needs no restart. ``` The message distinguishes cases that share a status code — `404` from -`/dataset` means "not in the cache", `404` from `/sync` means "not in -`sources.yaml`", and a `404` from `DELETE .../hosts/{hostname}` names whichever +`/dataset` means "not in the cache", `404` from `/sync` means "not +configured", and a `404` from `DELETE .../hosts/{hostname}` names whichever of the two is missing. Treat the text as loggable, not matchable: branch on the status code, read the message. @@ -167,7 +169,7 @@ be routed at all. An unmatched `?group=` is still an empty result. `DELETE /api/v1/sources/{id}` drops the cached entry and reports how many hosts went with it. It removes **cached data, not configuration**: a source -still listed in `sources.yaml` will be refilled by its next scheduled sync or +still configured will be refilled by its next scheduled sync or an explicit `POST .../sync`. The point is a source you have *removed* from config, whose entry would otherwise be served — and re-persisted in snapshots — until the next restart. @@ -425,18 +427,47 @@ be a real number, boolean, or nested structure. --- +## Connectors (admin-only) + +What each [connector](connectors.md#connectors-and-sources) runs, which sources +use it, and how they are doing — restricted keys always get `403`, since a +connector names a script, its project and its arguments. + +| Route | Meaning | +|---|---| +| `GET /api/v1/connectors` | Every connector item, then the two built-ins (`remote`, `static_inventory`, flagged `built_in: true`) | +| `GET /api/v1/connectors/{id}` | One of them; `404` when it is neither an item nor a built-in | + +Each carries `connector_id`, `name` (the description, or the id), +`connector_type` (`script`, `ssh`, `remote`, `static_inventory`), `built_in`, +`project_id`, `path`, `script_args`, `host_args`, `output_format` (null for ssh +and the built-ins), `timeout_seconds` (null for the built-ins), `used_by` — each +source naming it, with `source_id`, `name`, the `timeout_seconds` it actually +syncs under (its override or the connector's), `failing` and `synced` — and +`health`, those sources' sync health counted: `sources`, `healthy`, `failing`, +`never_synced`. A broken script reads as a connector whose sources all fail at +once. Full example in [connectors](connectors.md#inspecting-connectors-apiv1connectors). + +Connectors are additive to what the API always answered: `GET /api/v1/sources/{id}` +names the source's connector as `gathering.connector`, and `gathering.project_id` +is null for remote and ssh sources (nothing runs from a checkout); `GET +/api/v1/projects` lists under `used_by.connectors` the connectors whose scripts +live in each checkout; and `GET /api/v1/` links `connectors` for an admin key. + +--- + ## Projects (admin-only) Operational routes for git project checkouts — restricted keys always get `403`. | Route | Meaning | |---|---| -| `GET /api/v1/projects` | Configured projects with `checkout_present`, their sync settings and `sync_health` | +| `GET /api/v1/projects` | Configured projects with `checkout_present`, their sync settings, `sync_health`, and `used_by` (the `connectors`, `sources`, `enrichers` and `endpoints` that run from the checkout — `connectors` is new in 0.35) | | `POST /api/v1/projects/{id}/sync` | Clone/update the checkout to the branch tip, on demand | The sync route is how a pipeline in the scripts repository rolls new script versions without restarting the app (see -[configuration → projects.yaml](configuration.md#projectsyaml)): `200` with the +[configuration → projects](configuration.md#projects)): `200` with the duration on success, `502` when git fails (bad URL, auth, network), `404` for an unknown project id. Scripts are re-read from disk on every execution, so an updated checkout takes effect on the next run. @@ -451,10 +482,10 @@ files describe the estate. | Route | Meaning | |---|---| -| `GET /api/v1/config` | Every file with its sha256, what is missing, whether it still loads, whether the process is running it | -| `GET /api/v1/config/{file}` | One file verbatim; `ETag` is the sha256, `If-None-Match` answers `304` | -| `PUT /api/v1/config/{file}` | Replace one file (raw YAML body) | -| `DELETE /api/v1/config/{file}` | Remove one optional file | +| `GET /api/v1/config` | Every file (at any depth) with its sha256, what is missing (only ever `config.yaml`), whether it still loads, whether the process is running it | +| `GET /api/v1/config/{path}` | One file verbatim, by its path with slashes (`sources/dc1/src-d42.yaml`); `ETag` is the sha256, `If-None-Match` answers `304` | +| `PUT /api/v1/config/{path}` | Create or replace one file (raw YAML body); folders are created | +| `DELETE /api/v1/config/{path}` | Remove one file, and the folders it leaves empty | | `PUT /api/v1/config` | Replace the whole directory in one transaction | | `POST /api/v1/config/validate` | Dry run — the same checks as `--check-config`, nothing written | | `POST /api/v1/config/reload` | Apply what is on disk to the running process | @@ -463,7 +494,8 @@ Writes take `?reload=true` to apply the change as part of the write. Status codes: `400` the directory would not load (body carries `errors`, an array of every problem, alongside the usual `error`), `409` it would break the running process (a key's env var is missing, or authentication would be turned off), -`412` `If-Match` did not match, `404` not a configuration file. +`412` `If-Match` did not match, `404` not a configuration path. The summary and +reload report count `connectors` beside the other sections. Nothing is written unless the whole proposed directory validates, and nothing is applied unless it can be applied completely. Full contract, including which diff --git a/docs/architecture.md b/docs/architecture.md index d46c609..f45f884 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,7 +74,8 @@ Adapters come in two kinds, distinguished by which way the call travels across t 1. `http::sync::sync_source` resolves the `Source` and builds a `SyncScope` (full / host / group) from query params 2. `application::sync::sync_source` resolves credentials via `SecretsPort`, - runs the right `ConnectorPort` (script or SSH, per `source.connector_type`), + runs the right `ConnectorPort` (script, SSH, static inventory or remote, per + the `connector_type` the source resolved from its connector at load), and applies the resulting `Dataset` to the cache according to scope and `sync_mode` (replace / merge) 3. The handler maps the `SyncOutcome` to a `SyncResult` JSON body diff --git a/docs/browsable-api.md b/docs/browsable-api.md index 8a69de2..469864c 100644 --- a/docs/browsable-api.md +++ b/docs/browsable-api.md @@ -249,8 +249,9 @@ login form and a session cookie rather than a header box. `allows_source` per id. **The UI inherits the permission model for free**, and a restricted key browsing the UI sees exactly the sources its token allows. - A reload that removes a key must invalidate its sessions, or the console - outlives the revocation. The registry replace sweeps the session map by key - name. + outlives the revocation. The registry replace sweeps the session map by the + key's `name` (its identity), never by its label: two keys may share a + description. If no keys are configured at all, the API is open (`auth.rs:213`) and so is the UI, with the same loud warning at startup. No special case. @@ -266,11 +267,11 @@ would have been a report. So the detail pages carry forms: | Page | Action | Route | |---|---|---| -| source detail | Sync now | `POST /api/v1/sources/{id}/sync` | +| source detail | Sync | `POST /api/v1/sources/{id}/sync` | | source detail | Evict cache | `DELETE /api/v1/sources/{id}` | | host detail | Delete host | `DELETE /api/v1/sources/{id}/hosts/{host}` | -| enricher detail | Run now | `POST /api/v1/enrichers/{id}/run` | -| project detail | Sync now | `POST /api/v1/projects/{id}/sync` | +| enricher detail | Run | `POST /api/v1/enrichers/{id}/run` | +| project detail | Sync | `POST /api/v1/projects/{id}/sync` | Two consequences, both mandatory: @@ -330,6 +331,8 @@ on its own merits. paginated envelope with a one-entry `hostvars` map. The detail page needs a resource, and consumers that fetch one host want it too. - **`GET /api/v1/`** — the index: every collection, plus counts, plus links. + The admin-only collections (`projects`, `connectors`, `config`) are linked + only for an admin key, by the same rule as `related`. ### The gap: a source that never synced is invisible diff --git a/docs/caching.md b/docs/caching.md index 7c1923e..979f42e 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -181,6 +181,6 @@ writes never wait on disk. ## Memory notes Entries are only removed by `CachePort::remove` (currently unused by any route) or -process restart; an entry whose source disappears from `sources.yaml` survives in +process restart; an entry whose source disappears from the configuration survives in the cache until then. With inventory-sized payloads this is harmless, but it's worth knowing when reading `GET /api/v1/sources` output. diff --git a/docs/cli.md b/docs/cli.md index a29127b..2e51556 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,12 +1,14 @@ # Command-line interface (CLI) Unified API is a single binary — all runtime behaviour is controlled through -environment variables and YAML configuration files, plus one flag -(`--check-config`) for validating configuration without starting anything. This +environment variables and YAML configuration files, plus two flags: +`--check-config` validates configuration without starting anything, and +`--migrate-config` converts a 0.34 configuration directory to item files. This page documents how to start, stop and operate the service from the command line. - [Running the binary](#running-the-binary) - [Validating configuration (`--check-config`)](#validating-configuration---check-config) +- [Converting a 0.34 directory (`--migrate-config`)](#converting-a-034-directory---migrate-config) - [Environment variables](#environment-variables) - [Log level tuning](#log-level-tuning) - [Health checks](#health-checks) @@ -50,8 +52,8 @@ vars, so missing credentials are **not** an error here). ```bash CONFIG_DIR=./config unified-api --check-config -# configuration OK (./config): 4 sources, 1 views, 2 credentials, ... → exit 0 -# configuration INVALID (./config): unknown field `porT` ... → exit 1 +# configuration OK (./config): 4 sources, 3 connectors, 1 views, 2 credentials, ... → exit 0 +# configuration INVALID (./config): sources/a.yaml: source 'src-a': unknown field `ttl_second` ... → exit 1 ``` Exit `0` means this directory would start; exit `1` prints every validation @@ -66,6 +68,51 @@ deploy: ghcr.io/opusprojects/unified-api:latest --check-config ``` +On a directory still in the **0.34 layout** (`sources.yaml` and the other fixed +files, keyed by id), `--check-config` does not fail and does not write: it +converts the directory in a scratch copy, validates THAT, and says so — + +``` +configuration OK (./config) in the 0.34 layout: startup would convert it (converted +credentials.yaml, sources.yaml, ... into 15 item files; connector 'd42' extracted from: +src-d42-a, src-d42-b; ...); run --migrate-config to do it now +``` + +— because the question CI asks is whether this version will start on the +directory, and since startup migrates, it will exactly when the conversion +loads. The checkout may well be mounted read-only; nothing is written to it. + +--- + +## Converting a 0.34 directory (`--migrate-config`) + +```bash +unified-api --migrate-config ./config # or the directory in CONFIG_DIR, or ./config +``` + +Converts a directory in the 0.34 layout to item files in place, prints what it +did, and exits — the same code startup runs on its own (see +[configuration → upgrading](configuration.md#upgrading-from-034)), offered as a +command so a configuration **repository** can be converted once and committed. +A repository and a pod upgraded in place end up with the same files. + +``` +converted credentials.yaml, endpoints.yaml, enrichers.yaml, projects.yaml, sources.yaml into 15 item files +connector 'd42' extracted from: src-d42-a, src-d42-b +connector 'ssh-gather-facts' extracted from: src-ssh-dc1 +source 'src-ssh-dc1': its project_id was dropped — an ssh connector runs nothing from a checkout +migrated ./config; the old files are in ./config/.legacy-config-20261025T185536Z +``` + +- The old files move to `.legacy-config-/`, hidden so the loader + skips it. Comments are not carried into the new files (they are not data); + they are in the backup. Commit the new files and delete the backup when done. +- Nothing moves unless the converted directory loads: a failure prints + `migration of ./config FAILED, nothing was changed: ...` and exits `1`. +- A directory already in the item-file layout prints `... is already in the + item-file layout; nothing to do` and exits `0`, so the command is safe to + run twice. + --- ## Environment variables @@ -75,12 +122,12 @@ variables. None are mandatory — the defaults are safe for local development. | Variable | Default | Description | |---|---|---| -| `CONFIG_DIR` | `config` (relative to CWD) | Path to the directory containing the YAML configuration files | +| `CONFIG_DIR` | `config` (relative to CWD) | Path to the configuration directory (`config.yaml` plus item files) | | `UNIFIED_API_KEY` | *(unset — auth disabled)* | Static API key for `/api/v1/*` routes. Compared in constant time. Health, metrics and Swagger remain public | | `RUST_LOG` | `info` | Log level filter ([`tracing` / `EnvFilter`](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html) syntax) | -Credentials are resolved from additional env vars whose names come from -`credentials.yaml` — see [configuration](configuration.md) for the +Credentials are resolved from additional env vars whose names come from the +`credentials` items — see [configuration](configuration.md) for the `env_prefix`/`secret_keys` pattern. --- @@ -195,8 +242,8 @@ The process exits with one of two codes, so a supervisor can tell a clean shutdo | Code | Meaning | |---|---| -| `0` | Clean shutdown (SIGTERM / Ctrl-C), or `--check-config` found the configuration valid | -| `1` | Fatal startup error — configuration file missing or invalid, bind failure, etc. The error is logged to stderr. Also `--check-config` with an invalid configuration | +| `0` | Clean shutdown (SIGTERM / Ctrl-C), `--check-config` found the configuration valid, or `--migrate-config` converted the directory (or had nothing to do) | +| `1` | Fatal startup error — configuration file missing or invalid, a 0.34 directory whose conversion would not load, bind failure, etc. The error is logged to stderr. Also `--check-config` with an invalid configuration, and a failed `--migrate-config` | --- diff --git a/docs/config-api.md b/docs/config-api.md index 7e161fd..4fbcd59 100644 --- a/docs/config-api.md +++ b/docs/config-api.md @@ -6,7 +6,7 @@ over HTTP, so a configuration-as-code pipeline can **push** a change to each instance instead of publishing an artifact each instance has to **pull**. It is **off by default**. Turning it on means an admin key can rewrite every -file the loader reads — `api_keys.yaml` included, which is the same authority +file the loader reads — the `api_keys` items included, which is the same authority as editing the directory the container mounts. ```yaml @@ -53,10 +53,10 @@ not set — with a body that says so rather than a bare `404`. | Route | Meaning | |---|---| -| `GET /api/v1/config` | The directory: every file with size, sha256 and mtime, what is missing, whether it still loads, whether the process is running it | -| `GET /api/v1/config/{file}` | One file, verbatim. `ETag` is the sha256; `If-None-Match` answers `304` | -| `PUT /api/v1/config/{file}` | Replace one file. Body is raw YAML | -| `DELETE /api/v1/config/{file}` | Remove one optional file | +| `GET /api/v1/config` | The directory: every file (at any depth) with size, sha256 and mtime, what is missing, whether it still loads, whether the process is running it | +| `GET /api/v1/config/{path}` | One file, verbatim. `ETag` is the sha256; `If-None-Match` answers `304` | +| `PUT /api/v1/config/{path}` | Create or replace one file (its folders are created). Body is raw YAML | +| `DELETE /api/v1/config/{path}` | Remove one file (folders it leaves empty go too) | | `PUT /api/v1/config` | Replace the whole directory in one transaction | | `POST /api/v1/config/validate` | Dry run: same checks, nothing written, ever | | `POST /api/v1/config/reload` | Apply what is on disk to the running process | @@ -64,22 +64,41 @@ not set — with a body that says so rather than a bare `404`. `PUT` and `DELETE` take `?reload=true` to apply the change as part of the write, so a push is one request rather than two. -The files are exactly the ones the loader reads: +`{path}` is a file's path inside the directory, **slashes included** — the +directory is item files in folders (see [configuration](configuration.md#the-directory)): -``` -config.yaml credentials.yaml sources.yaml views.yaml -enrichers.yaml projects.yaml endpoints.yaml api_keys.yaml +```bash +curl -sS "$BASE/api/v1/config/sources/acme/src-d42-acme.yaml" -H "X-API-Key: $KEY" + +curl -sS -X PUT "$BASE/api/v1/config/connectors/d42.yaml?reload=true" \ + -H "X-API-Key: $KEY" -H 'Content-Type: application/yaml' \ + --data-binary @config/connectors/d42.yaml ``` -Anything else is a `404` naming the list. `config.yaml` cannot be deleted. +A path must be one the loader would read: relative, segments of letters, +digits, `-`, `_` and `.`, none hidden, ending in `.yaml` or `.yml`. Anything else +is a `404` naming that rule — the same rule the loader applies to what it finds, +so a file the API cannot name cannot be part of the configuration either. +`config.yaml`, at the root, cannot be deleted, and it is the only file `missing` +in `GET /api/v1/config` can ever list: every other file is an item file that is +there or not. + +A file still in the **0.34 layout** (items keyed by id, as `sources.yaml` used to +be) is refused like any other unknown top-level key, with a hint naming +`unified-api --migrate-config` — a pipeline that has not converted its +repository yet gets a one-line fix rather than a typo hunt. The startup +migration (see [configuration → upgrading](configuration.md#upgrading-from-034)) +converts a directory on disk; it does not convert what is pushed. --- ## Pushing a whole directory -The verb a pipeline wants. `prune: true` makes the directory become *exactly* -the payload — the same semantics as the configuration image it replaces, where -what is not in the image is not in `/config`. +The verb a pipeline wants. `files` is keyed by path relative to the directory, +and `prune: true` makes the directory become *exactly* the payload — the same +semantics as the configuration image it replaces, where what is not in the image +is not in `/config`. Every file the payload leaves out, at any depth, is +removed, and so are the folders that leaves empty; `deleted` names them. ```bash curl -sS -X PUT "$BASE/api/v1/config?reload=true" \ @@ -90,9 +109,10 @@ curl -sS -X PUT "$BASE/api/v1/config?reload=true" \ { "prune": true, "files": { - "config.yaml": "server:\n host: \"0.0.0.0\"\n port: 8080\n", - "sources.yaml": "src-dc4:\n name: \"DC4\"\n project_id: \"prj-inv\"\n ...\n", - "projects.yaml": "prj-inv:\n name: \"Inventory\"\n git_url: \"...\"\n" + "config.yaml": "server:\n host: \"0.0.0.0\"\n port: 8080\n", + "connectors/d42.yaml": "connectors:\n - name: d42\n project: prj-inv\n path: d42/inventory.py\n", + "sources/dc4/src-dc4.yaml": "sources:\n - name: src-dc4\n description: \"DC4\"\n connector: d42\n ...\n", + "projects/prj-inv.yaml": "projects:\n - name: prj-inv\n git_url: \"...\"\n" } } JSON @@ -100,32 +120,53 @@ JSON ```json { - "written": ["config.yaml", "projects.yaml", "sources.yaml"], + "written": ["config.yaml", "connectors/d42.yaml", "projects/prj-inv.yaml", + "sources/dc4/src-dc4.yaml"], + "deleted": ["sources/src-old.yaml"], "etag": "9f2c…", - "summary": {"sources": 1, "views": 0, "credentials": 0, "enrichers": 0, - "endpoints": 0, "projects": 1, "api_keys": 0}, + "summary": {"sources": 1, "connectors": 1, "views": 0, "credentials": 0, + "enrichers": 0, "endpoints": 0, "projects": 1, "api_keys": 0}, "reloaded": { "generation": 3, - "applied": ["sources"], - "sources": {"added": ["src-dc4"], "changed": ["src-old"]}, + "applied": ["sources", "connectors"], + "sources": {"added": ["src-dc4"], "removed": ["src-old"]}, + "connectors": {"added": ["d42"]}, "api_keys": 2 }, "reload_pending": false } ``` +A pipeline builds that payload from its checkout of the configuration +repository — every YAML file, keyed by its path relative to the directory: + +```bash +cd config +find . -type f \( -name '*.yaml' -o -name '*.yml' \) -not -path '*/.*' | sed 's|^\./||' | sort | + while read -r f; do jq -n --arg p "$f" --rawfile c "$f" '{($p): $c}'; done | + jq -s '{prune: true, files: add}' > /tmp/payload.json + +curl -sS -X PUT "$BASE/api/v1/config?reload=true" \ + -H "X-API-Key: $KEY" -H 'Content-Type: application/json' \ + --data-binary @/tmp/payload.json +``` + +Hidden paths are left out because the loader skips them too (`.git`, editor +swap files); a file the push should not carry is one the directory should not +hold. + Validate first from CI, against the instance that will run it, without writing anything: ```bash curl -sS -X POST "$BASE/api/v1/config/validate" \ -H "X-API-Key: $KEY" -H 'Content-Type: application/json' \ - -d '{"files": {"sources.yaml": "..."}, "prune": true}' + --data-binary @/tmp/payload.json ``` ```json {"valid": false, "errors": [ - "Source 'src-dc4' references unknown project 'prj-typo'", + "Connector 'd42' references unknown project 'prj-typo'", "View 'view-all' references unknown source 'src-gone'" ]} ``` @@ -151,11 +192,12 @@ is already running. | | Effect | |---|---| -| `sources.yaml` | New sources start syncing, removed ones stop, changed ones restart on the new settings | -| `views.yaml`, `enrichers.yaml`, `endpoints.yaml` | Live on the next request/tick | -| `projects.yaml` | Pull tasks restart; a project that arrived with the reload is cloned in the background | -| `credentials.yaml`, `secrets:` | The resolver chain is rebuilt (which also drops the resolution cache) | -| `api_keys.yaml` | In force on the next request | +| `sources` | New sources start syncing, removed ones stop, changed ones restart on the new settings | +| `connectors` | A source is resolved through its connector, so the sources using a changed connector count as changed themselves and restart on it | +| `views`, `enrichers`, `endpoints` | Live on the next request/tick | +| `projects` | Pull tasks restart; a project that arrived with the reload is cloned in the background | +| `credentials`, `secrets:` | The resolver chain is rebuilt (which also drops the resolution cache) | +| `api_keys` | In force on the next request | | `server.readyz_require_all_sources` | Next probe | | `server.refresh_timeout_seconds`, `server.refresh_max_concurrent` | Next refresh. A shrunk concurrency cap lets in-flight refreshes finish under the old limit and reclaims their permits as they land | | `server.shutdown_grace_seconds` | The next shutdown — the drain reads whatever a reload set last | @@ -199,8 +241,9 @@ source are already serialised. - **Validated as a directory, before anything moves.** The proposed set is staged in a temporary directory and loaded there. A rejected change never touched the real one. -- **Rejected whole.** Cross-file references (a source naming a project, a view - naming a source, a restricted key naming an endpoint) only make sense +- **Rejected whole.** Cross-file references (a source naming a connector, a + connector naming a project, a view naming a source, a restricted key naming an + endpoint) only make sense against the complete set, so the complete set is what is accepted or refused. - **Atomic per file.** Every file is written to a temporary name and renamed into place, so no reader ever sees a half-written file. @@ -209,7 +252,7 @@ source are already serialised. it under a running process is refused with `409`. Write the file and restart if that is really the intent. - **A key that cannot be resolved fails the whole operation.** If - `api_keys.yaml` names an env var that is not set, the write is refused + an `api_keys` item names an env var that is not set, the write is refused **before** it is committed, rather than landing and then locking a consumer out. - **A write without `?reload=true` says so.** `reload_pending: true` means the @@ -242,7 +285,7 @@ a single pipeline and stays unceremonious. ## Permissions and audit -Admin keys only, **reads included** — `config.yaml` and `credentials.yaml` +Admin keys only, **reads included** — `config.yaml` and the credentials describe the estate (which systems exist, which variable holds which credential), which is exactly what a restricted consumer key has no business enumerating. @@ -251,7 +294,7 @@ Every write and reload emits an audit line under the `audit` target, with the key that did it and the request id: ``` -actor=pipeline action=config_write_reload resource=sources.yaml request_id=req-42 outcome=success +actor=pipeline action=config_write_reload resource=sources/dc4/src-dc4.yaml request_id=req-42 outcome=success ``` `RUST_LOG=audit=info` to isolate them. See [observability](observability.md). @@ -273,6 +316,12 @@ process and then lost the next time the container starts from its image. An instance whose directory is read-only will refuse writes with a `500` naming the filesystem error; leave `config_api.enabled` off there. +One exception: a read-only directory still in the 0.34 layout is converted at +startup into a private temporary copy, and that copy is what the process runs — +so writes *succeed*, land in the copy, and are gone at the next restart (which +converts the read-only files again). Convert the source of the files first; see +[configuration → upgrading](configuration.md#a-read-only-directory). + --- ## Metrics diff --git a/docs/configuration.md b/docs/configuration.md index 57b522f..56b235a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,22 +1,28 @@ # Configuration -Configuration is split across YAML files in a directory (default `./config`, -override with the `CONFIG_DIR` environment variable). Only `config.yaml` is -mandatory; every other file defaults to empty if absent. - -All cross-references are **validated at startup** — an unknown source, credential -or project id anywhere in the config aborts boot with a list of every error found. -The same validation runs standalone with `unified-api --check-config` +Everything is YAML in one directory (default `./config`, override with the +`CONFIG_DIR` environment variable), laid out the way AWX config as code is: +`config.yaml` at the root holds the server settings, and every other `*.yaml` / +`*.yml` file, **at any depth**, holds items under a typed top-level key. +Only `config.yaml` is mandatory; a directory with nothing else is a valid, +empty inventory. + +All cross-references are **validated at startup** — an unknown source, connector, +credential or project anywhere in the config aborts boot with a list of every +error found. The same validation runs standalone with `unified-api --check-config` (see [CLI](cli.md#validating-configuration---check-config)) — for CI on a config repository. +- [The directory](#the-directory) - [config.yaml (required)](#configyaml-required) -- [sources.yaml](#sourcesyaml) -- [views.yaml](#viewsyaml) -- [credentials.yaml](#credentialsyaml) -- [enrichers.yaml](#enrichersyaml) -- [endpoints.yaml](#endpointsyaml) -- [projects.yaml](#projectsyaml) -- [api_keys.yaml](#api_keysyaml) +- [sources](#sources) +- [connectors](#connectors) +- [views](#views) +- [credentials](#credentials) +- [enrichers](#enrichers) +- [endpoints](#endpoints) +- [projects](#projects) +- [api_keys](#api_keys) +- [Upgrading from 0.34](#upgrading-from-034) - [Environment variables](#environment-variables) Configuration is read at startup, and can also be **written and reloaded over @@ -24,9 +30,111 @@ HTTP** where that is enabled — see [Configuration API](config-api.md). --- +## The directory + +``` +/etc/unified-api/ +├── config.yaml # required, at the root, under this name +├── sources/ # folders are for people: any names, any depth, or none +│ ├── acme/ +│ │ ├── src-d42-acme.yaml +│ │ └── src-ssh-acme.yaml +│ └── src-netbox.yaml +├── connectors/ +│ ├── d42.yaml +│ └── ssh-facts.yaml +├── views/ +│ └── vw-facts-all.yaml +├── credentials/ +│ └── cred-d42.yaml +├── projects/ +│ └── prj-inventory.yaml +└── api_keys/ + └── key-awx.yaml +``` + +| Top-level key | Holds | +|---|---| +| `sources` | Each inventory instance: its connector, credentials, config, schedule and TTLs | +| `connectors` | How to talk to a kind of system: the script (or SSH command), its project, its arguments | +| `views` | Read-only composites over several sources, routed by declared ownership | +| `endpoints` | Outputs combining cached datasets through a transformer | +| `enrichers` | Post-processors over data already in the cache | +| `projects` | Git repositories holding scripts and static inventories | +| `credentials` | Where each credential's secrets are read from (never the values) | +| `api_keys` | API keys by name and role (the values come from env vars) | + +Every item file looks the same: a key, and a list of items under it. + +```yaml +# sources/acme/src-d42-acme.yaml +sources: + - name: src-d42-acme # identity: URLs, references, cache keys + description: "Device42 Acme" # label, for people + connector: d42 + credentials: [cred-d42] + sync_interval_seconds: 300 + ttl_seconds: 3600 +``` + +The rules, and why they are what they are: + +- **The key says what an item is, never the folder.** A file in `connectors/` + holding `sources:` is read as a source. Folders exist for people — by team, by + estate, by datacenter — and may nest as deep as they like, or not exist at all. + Tying meaning to a folder would make moving a file a semantic change. +- **One item per file is the convention**, `
/.yaml`, and the one the + [upgrade](#upgrading-from-034) writes. It is not a rule: a file may hold several + items, of several types. A review of "add a tenant" is then one new file, not a + hunk in the middle of a 2,000-line map. +- **`name` is the identity**, as in AWX: what other items reference, what URLs + (`/api/v1/sources/{id}`) and cache keys use. Letters, digits, `-`, `_` and `.`, + not starting with a dot — it appears in URLs and in file names. The values are + the same ids 0.34 used as map keys, so nothing that reads the API changes. +- **`description` is optional, for people.** Every API response field called + `name` answers the description, falling back to the identity when there is + none — so responses read exactly as they did before item files. +- **A `name` defined twice is an error naming both files.** Which one won would + otherwise depend on the order files happen to be read in. +- **References drop the `_id`:** a source names its `connector` and + `credentials`, an enricher its `target`, `source` and `project`, an endpoint its + `sources` and `project`, a project its `credential`. +- **Hidden files and folders are skipped** (editor swap files, `.git`, the config + API's staging folders, the upgrade's backup). **Symlinks are followed**: a + Kubernetes ConfigMap mount is nothing but links into a hidden `..data` folder, + and skipping them would load an empty configuration. +- **A path may only use letters, digits, `-`, `_` and `.`** in each segment, none + starting with a dot, ending in `.yaml` or `.yml` — the same rule the + [config API](config-api.md) applies to what it is asked to write, so a file the + API cannot name cannot be part of the configuration either. + +Parsing is **strict**. Every way a file can be wrong fails startup naming the +file, and every error is reported at once, so a push with three typos in three +files names all three: + +| Mistake | Error | +|---|---| +| An unknown top-level key | `sources/a.yaml: unknown key 'sourcse' (expected one of: sources, views, ...)` | +| A 0.34 file pushed as-is | the same, ending `— this is the 0.34 layout (items keyed by id); convert it with unified-api --migrate-config ` | +| An unknown key inside an item | ``sources/a.yaml: source 'src-a': unknown field `ttl_second`, expected one of ...`` | +| An item without `name` | `sources/a.yaml: sources item 2 needs a 'name', its identity` | +| A name defined twice | `sources/b.yaml: source 'src-a' is already defined in sources/a.yaml` | + +A typo'd key failing loudly is the point: silently applying the default for +`ttl_second:` is how an inventory ends up an hour stale with nobody knowing why. +The free-form `config:` maps are the exception — they are arbitrary by design, +and their scalars are taken as text, exactly as written (`concurrency: 20` +arrives as `"20"`, `version: 1.10` as `"1.10"`, `account: 0x1F` as `"0x1F"`), since +they become environment variables and script settings. Every other text field +reads an unquoted scalar the same way: `script_args: ["--days", 7]` is +`["--days", "7"]`. + +--- + ## config.yaml (required) The one file that must exist, holding the `server`, `cache`, `projects` and `secrets` settings. +It is always at the root, under exactly this name, and holds no items. ```yaml server: @@ -63,6 +171,10 @@ server: # naming this key. Size it for the biggest configuration push (a whole # directory is one body) or host-vars write you expect. max_body_bytes: 2097152 + # Optional. This instance's address, as people and other instances reach it + # (the ingress, not the port above). A central sends it with the federation + # map, so each edge's Topology page links back to it. Reloadable. + url: "https://unified-api-central.example.com" # Optional. Absent = purely in-memory cache (restarts start empty). # With the block, the cache is snapshotted to `path` every `interval_seconds` @@ -72,7 +184,7 @@ cache: path: "/var/lib/unified-api/cache.json" interval_seconds: 60 -# Optional. Where git projects are cloned (one subdirectory per project id). +# Optional. Where git projects are cloned (one subdirectory per project name). projects: dir: "/var/lib/unified-api/projects" # default "projects" @@ -88,7 +200,7 @@ secrets: # Optional, OFF by default. Lets an admin key read, write and reload this # directory over HTTP, so a pipeline can push a change instead of publishing # an artifact the instance has to pull. Turning it on means an admin key can -# rewrite every file the loader reads, api_keys.yaml included — the same +# rewrite every file the loader reads, the api_keys items included — the same # authority as editing the mounted directory. See docs/config-api.md. config_api: enabled: false @@ -96,59 +208,155 @@ config_api: --- -## sources.yaml +## sources -One entry per inventory source; the key is the source id used in URLs. +One item per inventory **instance**: which system, reached how (its +`connector`), as whom (`credentials`), with which settings (`config`), how often, +and how long the data stays fresh. The name is the source id used in URLs. ```yaml -src-section9: - name: "Section 9 Inventory" - project_id: "prj-connectors-infra" # must exist in projects.yaml - script_path: "tests/adapters/out/connectors/inventory.py" - script_args: [] # CLI args for the script, e.g. ["--list"] - host_args: [] # CLI args for a HOST-scoped sync, e.g. - # ["--host", "{target}"] — replaces - # script_args for that run - host_sync_updates_group_vars: false # may a host-scoped sync take the vars of - # the groups it joins? (default false) - output_format: "native" # "native" (default) or "ansible" - connector_type: "script" # "script" (default) or "ssh" - sync_mode: "replace" # "replace" (default) or "merge" - credential_ids: ["cred-section9-api"] # must exist in credentials.yaml - sync_interval_seconds: 60 # background sync; 0/absent = manual only - # schedule: "30 2 * * *" # OR a cron cadence (UTC) — pick one - ttl_seconds: 3600 # dataset-level TTL - timeout_seconds: 300 # abort a sync that runs longer (default 300) - ttl_overrides: # optional per-group / per-host TTLs - groups: - production: 900 - hosts: - motoko.section9.net: 300 - allow_on_demand_refresh: false # may a READ trigger a sync? (default false) - config: # free-form strings passed to the script - scenario: "large" +# sources/src-section9.yaml +sources: + - name: src-section9 + description: "Section 9 Inventory" + connector: "sample-inventory" # a connector item, or a built-in (below) + credentials: ["cred-section9-api"] # credential items + sync_mode: "replace" # "replace" (default) or "merge" + sync_interval_seconds: 60 # background sync; 0/absent = manual only + # schedule: "30 2 * * *" # OR a cron cadence (UTC) — pick one + ttl_seconds: 3600 # dataset-level TTL (required) + # timeout_seconds: 900 # override the connector's bound, this source only + ttl_overrides: # optional per-group / per-host TTLs + groups: + production: 900 + hosts: + motoko.section9.net: 300 + host_sync_updates_group_vars: false # may a host-scoped sync take the vars of + # the groups it joins? (default false) + allow_on_demand_refresh: false # may a READ trigger a sync? (default false) + config: # free-form strings passed to the connector + scenario: "large" ``` | Field | Notes | |---|---| -| `script_path` | Executable run by the script connector, or remote command/facts selector for SSH | -| `script_args` | CLI arguments passed verbatim (no shell). `["--list"]` makes standard Ansible dynamic inventory scripts work unmodified. For SSH sources they are appended to the remote command in `script` gather mode | -| `host_args` | CLI arguments for a sync that names hosts, replacing `script_args` for that run, with `{target}` substituted by the requested hostnames (comma-joined). Empty = call the script exactly as a full sync would and filter the answer, which is what Ansible's own `--host` usually did anyway. No convention is imposed: `["--host", "{target}"]` is Ansible's contract, and a script that can return a host *with its groups* names its own flag. Script sources only — refused at startup elsewhere. See [connectors](connectors.md#asking-for-one-host-on-the-command-line-host_args) | +| `connector` | Required. The name of a [connector](#connectors) item, or a built-in: `remote` (another unified-api, see [federation](federation.md)) or `static_inventory` (an Ansible YAML inventory in a project checkout). See [connectors](connectors.md) | +| `credentials` | Credential items resolved before each sync and handed to the connector | +| `project`, `path` | `static_inventory` sources only: the project whose checkout holds the inventory, and the file inside it. Refused on every other source, where they would look like they select a script and would not | +| `timeout_seconds` | Overrides the connector's `timeout_seconds` for this source only — the bigger tenant of the same system. Absent = the connector's (default 300). A timed-out sync fails with a clear error instead of hanging its scheduler task or HTTP request | | `host_sync_updates_group_vars` | Whether a host-scoped sync also takes the vars of the groups it places those hosts in, and only those groups. Off by default: a granular call answers about a host, not about what a group means for its other members | -| `output_format` | What the script prints: `native` (the Dataset shape) or `ansible` (standard dynamic inventory JSON with `_meta` — converted on the fly, see [connectors](connectors.md)) | -| `connector_type` | `script` runs a local process; `ssh` fans out over hosts; `static_inventory` parses an Ansible YAML inventory (+ `group_vars/`/`host_vars/`) from disk natively; `remote` federates another unified-api instance (see [connectors](connectors.md)) | | `sync_mode` | How a **full** sync lands in the cache: `replace` swaps the dataset, `merge` patches it | | `ttl_*` | See [caching](caching.md) for the freshness model | | `allow_on_demand_refresh` | Whether `GET /dataset?host=X&refresh=true` may gather the named hosts. Off by default: a read that can cause SSH into a datacenter is a capability. Turning it on makes `ttl_seconds` load bearing — it becomes the threshold at which a read pays for a gather — so review it first. See [on-demand refresh](on-demand-refresh.md) | -| `timeout_seconds` | Hard limit on connector execution; a timed-out sync fails with a clear error instead of hanging its scheduler task or HTTP request | | `schedule` | Cron cadence instead of an interval: standard 5-field cron, optional leading seconds field, evaluated in **UTC**. Validated at startup; mutually exclusive with a non-zero `sync_interval_seconds`. Cron sources get no startup jitter (their times are deliberate) but back off on failure like interval sources, by letting occurrences pass | | `hosts_from_source` | SSH sources only: dynamic host list from another source's cached dataset (`source` + optional `match_pattern.groups`/`hosts` + `connect_via`); mutually exclusive with `config.hosts` — see [connectors](connectors.md#dynamic-host-lists-hosts_from_source) | | `advertise_scope` | Explicit ownership claim served on `GET /sources/{id}/scope` (`groups` + `hosts`; must name at least one). Optional — an SSH source already advertises its `hosts_from_source` pattern; this overrides the derivation. The consumer side of federation reads it so view ownership stops being a duplicated truth | -| `config` | Arbitrary `key: value` strings the connector script receives as JSON. The SSH connector reads `hosts`, `port`, `concurrency`, `ssh_connect_timeout_seconds` (per-host, default 30), `gather_mode`, `fact_path`, `ssh_known_hosts` (host key verification — unset accepts any key, with a warning per sync) from here — see [connectors](connectors.md) | +| `config` | Arbitrary `key: value` strings the connector receives (`SOURCE_CONFIG` for a script). The SSH connector reads `hosts`, `port`, `concurrency`, `ssh_connect_timeout_seconds` (per-host, default 30), `gather_mode`, `fact_path`, `ssh_known_hosts` (host key verification — unset accepts any key, with a warning per sync) from here; a `remote` source reads `url` and `source` — see [connectors](connectors.md) | + +### Source or connector? + +What a source used to say about its script now lives on a connector, so ten +Device42 tenants are ten sources and **one** connector instead of ten copies of +the same script path, arguments and output format that drift apart one edit at a +time. The split follows one question — *does it describe the script, or this +instance of the system?* + +| Field | Lives on | Why | +|---|---|---| +| `type` (script / ssh) | connector | The protocol | +| `project`, `path` | connector | Where the script is | +| `script_args`, `host_args` | connector | How the script is called | +| `output_format` | connector | What the script prints | +| `timeout_seconds` | connector, overridable per source | A property of the script, except for the tenant ten times bigger | +| `credentials`, `config` | source | Which instance, as whom | +| `schedule`, `sync_interval_seconds`, `ttl_seconds`, `ttl_overrides` | source | How often and how fresh, per instance | +| `sync_mode`, `host_sync_updates_group_vars` | source | Cache policy of this instance's entry, not a property of the script | +| `hosts_from_source`, `allow_on_demand_refresh`, `advertise_scope` | source | Per instance | + +### Built-in connectors + +Two connectors need no item: the protocol is fixed, and the only thing to +configure — where — is per source. + +```yaml +sources: + # Another unified-api instance (federation) + - name: src-edge-dc2 + description: "Datacenter 2" + connector: remote + credentials: ["cred-edge-dc2"] # a token credential = the remote API key + sync_interval_seconds: 120 + ttl_seconds: 600 + config: + url: "https://unified-api-dc2.example.com" + source: "src-ssh" # the source id ON THE REMOTE instance + + # An Ansible YAML inventory in a project checkout + - name: src-linux + description: "Linux static inventory" + connector: static_inventory + project: "prj-inventories" # the checkout + path: "linux/inventory.yaml" # the file inside it + sync_interval_seconds: 300 + ttl_seconds: 600 +``` + +- `remote` requires `config.url` and `config.source`, and refuses `project` / + `path`. See [federation](federation.md). +- `static_inventory` requires `project` and `path` on the source itself. See + [connectors](connectors.md#static-inventory-sources-connector-static_inventory). + +`project` + `path` mean the same thing everywhere they appear — a file inside +that project's checkout — whether it is a connector's script or a static +source's inventory. + +--- + +## connectors + +How to talk to a kind of system. Full treatment, including the script contract, +in [connectors](connectors.md). + +```yaml +# connectors/d42.yaml +connectors: + - name: d42 + description: "Device42 inventory script" + type: "script" # script (default) | ssh + project: "prj-inventory" # the checkout `path` resolves in + path: "dynamic/d42/inventory.py" + script_args: ["--list"] # CLI args, verbatim (no shell) + host_args: ["--host", "{target}"] # replace script_args for a host-scoped sync + output_format: "ansible" # native (default) | ansible + timeout_seconds: 300 # default 300; a source may override it + +# connectors/ssh-facts.yaml +connectors: + - name: ssh-facts + description: "SSH fact gathering" + type: "ssh" + path: "gather_facts" # a label for the built-in gather modes, + # or the remote command for gather_mode: script +``` + +| Field | Notes | +|---|---| +| `type` | `script` (default) runs a local process; `ssh` connects to the fleet natively | +| `project` | The project whose checkout holds the script. Required for a script connector with a relative `path`; refused on ssh (nothing runs from a checkout) | +| `path` | script: the file to run, relative to the project checkout or absolute. ssh: the remote command for `gather_mode: script`, or a label such as `gather_facts` for the built-in modes | +| `script_args` | CLI arguments passed verbatim. `["--list"]` makes standard Ansible dynamic inventory scripts work unmodified. On ssh, appended to the remote command in `script` gather mode | +| `host_args` | Script only. CLI arguments for a sync that names hosts, replacing `script_args` for that run, `{target}` substituted by the requested hostnames (comma-joined). Empty = call the script as a full sync would and filter the answer. See [connectors](connectors.md#asking-for-one-host-on-the-command-line-host_args) | +| `output_format` | Script only. `native` (the Dataset shape) or `ansible` (dynamic inventory JSON with `_meta`, converted on the fly) | +| `timeout_seconds` | Bound on a sync through this connector (default 300); a source's own `timeout_seconds` overrides it | + +Settings that look active and are not are refused rather than ignored: `project`, +`host_args` and `output_format` on an ssh connector, a relative script `path` +without a `project`, and the names `remote` and `static_inventory`, which belong +to the built-ins. --- -## views.yaml +## views Read-only composites over several sources: one id that routes a per-host read to whichever member owns that host. Views answer on the **source routes**, share @@ -156,19 +364,21 @@ the source id space (a collision fails startup) and hold no cache entry of their own. Full treatment in [views](views.md). ```yaml -vw-facts-all: - name: "Facts, both datacenters" - members: # ordered: the first claim wins - - source: "src-ssh-dc1" # where the facts come from - owns: - source: "src-d42" # inventory the pattern resolves against - groups: ["datacenter_dc1"] - - source: "src-edge-dc2" # a remote (federated) member works too - owns: - source: "src-d42" - groups: ["datacenter_dc2"] - hosts: ["appliance01.dc2.example"] # claimed literally as well - # ttl_seconds: 30 # optional; absent = inherit the owner's +# views/vw-facts-all.yaml +views: + - name: vw-facts-all + description: "Facts, both datacenters" + members: # ordered: the first claim wins + - source: "src-ssh-dc1" # where the facts come from + owns: + source: "src-d42" # inventory the pattern resolves against + groups: ["datacenter_dc1"] + - source: "src-edge-dc2" # a remote (federated) member works too + owns: + source: "src-d42" + groups: ["datacenter_dc2"] + hosts: ["appliance01.dc2.example"] # claimed literally as well + # ttl_seconds: 30 # optional; absent = inherit the owner's ``` | Field | Meaning | @@ -186,16 +396,16 @@ sources often sync daily on purpose, and some hosts (appliances that take no SSH) never enter any cache at all — both are exactly the hosts on-demand refresh exists for. See [views → ownership](views.md#ownership-is-declared-not-inferred). -Unlike the other config files, an unknown key inside a view is a hard startup -error. Ownership is the routing table, and `grups:` instead of `groups:` would -otherwise parse as an empty pattern, which claims everything. +Strict parsing matters most here. Ownership is the routing table, and `grups:` +instead of `groups:` would otherwise parse as an empty pattern, which claims +everything. A view is granted to an API key exactly like a source, by its id under `sources:`; the key needs no access to the members. --- -## credentials.yaml +## credentials Credentials **never contain secrets** — they describe *where* to read them; this is the field reference, and the full resolution story (backends, the @@ -203,30 +413,32 @@ cache and rotation, native Vault, failure semantics) is [its own page](secrets.md). ```yaml -cred-section9-api: - name: "Section 9 API" - type: "username_password" # username_password | token | ssh_key - env_prefix: "SECTION9" # reads SECTION9_USERNAME, SECTION9_PASSWORD... - secret_keys: # our key -> env var suffix (or JSON field) - username: "USERNAME" - password: "PASSWORD" - -cred-ssh-infra: - name: "SSH Infrastructure" - type: "ssh_key" - env_prefix: "INFRA_SSH" - secret_keys: - username: "USERNAME" - file_keys: # our key -> file path; passed as _path - ssh_key: "/run/secrets/infra-ssh/id_rsa" - -cred-d42-vault: - name: "Device42 (from Vault)" - type: "username_password" - vault_path: "infra/device42" # KV v2 path under secrets.vault.mount - secret_keys: # our key -> field in the Vault secret - username: "user" # (omit secret_keys to take every field) - password: "pass" +# credentials/cred-section9-api.yaml +credentials: + - name: cred-section9-api + description: "Section 9 API" + type: "username_password" # username_password | token | ssh_key + env_prefix: "SECTION9" # reads SECTION9_USERNAME, SECTION9_PASSWORD... + secret_keys: # our key -> env var suffix (or JSON field) + username: "USERNAME" + password: "PASSWORD" + + - name: cred-ssh-infra + description: "SSH Infrastructure" + type: "ssh_key" + env_prefix: "INFRA_SSH" + secret_keys: + username: "USERNAME" + file_keys: # our key -> file path; passed as _path + ssh_key: "/run/secrets/infra-ssh/id_rsa" + + - name: cred-d42-vault + description: "Device42 (from Vault)" + type: "username_password" + vault_path: "infra/device42" # KV v2 path under secrets.vault.mount + secret_keys: # our key -> field in the Vault secret + username: "user" # (omit secret_keys to take every field) + password: "pass" ``` Resolution order per credential: `vault_path` (native Vault, see below), or @@ -261,7 +473,7 @@ becoming a request storm against Vault). --- -## enrichers.yaml +## enrichers Post-processors over data already in the cache. Two modes: @@ -269,74 +481,85 @@ Post-processors over data already in the cache. Two modes: returns a partial dataset (modified/removed hosts): ```yaml -enrich-resolve-ssh: - name: "Resolve SSH reachability" - target_id: "src-section9" # dataset being enriched - script_path: "enrichers/resolve.py" - script_args: [] # optional CLI args for the script - sync_interval_seconds: 300 # scheduled run; 0/absent = manual only - # schedule: "45 2 * * *" # OR a cron cadence (UTC) — pick one - timeout_seconds: 300 # abort a run that takes longer (default 300) - config: {} +# enrichers/enrich-resolve-ssh.yaml +enrichers: + - name: enrich-resolve-ssh + description: "Resolve SSH reachability" + target: "src-section9" # dataset being enriched + project: "prj-connectors" # optional: resolve script_path in this checkout + script_path: "enrichers/resolve.py" + script_args: [] # optional CLI args for the script + sync_interval_seconds: 300 # scheduled run; 0/absent = manual only + # schedule: "45 2 * * *" # OR a cron cadence (UTC) — pick one + timeout_seconds: 300 # abort a run that takes longer (default 300) + config: {} ``` **Declarative merge** — copies specified hostvars fields from one source into another, matched by hostname. No script needed: ```yaml -enrich-business-unit: - name: "Add business_unit from D42" - target_id: "src-ssh-dc1" # dataset being enriched - source_id: "src-d42" # dataset to copy fields from - fields: ["business_unit"] # top-level hostvars keys to copy - sync_interval_seconds: 3600 +# enrichers/enrich-business-unit.yaml +enrichers: + - name: enrich-business-unit + description: "Add business_unit from D42" + target: "src-ssh-dc1" # dataset being enriched + source: "src-d42" # dataset to copy fields from + fields: ["business_unit"] # top-level hostvars keys to copy + sync_interval_seconds: 3600 ``` +An enricher is not a connector — it gathers nothing — so it keeps its own +`script_path` rather than naming a connector. + --- -## endpoints.yaml +## endpoints Output endpoints combine one or more cached datasets through a transformer. Each endpoint sets **exactly one** of `output` (a builtin, in-process transformer) or `script_path` (an external script). ```yaml -# Builtin transformer — no script, no process spawn. -ep-ansible-full: - name: "Full Ansible Inventory" - source_ids: ["src-section9", "src-infra"] - output: ansible # builtins: ansible, json, csv - config: - filter: "datacenter=section9" # host, group, filter, fields, exclude_fields (see docs/endpoints.md) - -# A constructed inventory: merge everything, return only the hosts one source -# has — the others still enrich them and still group them. Optional; a limit -# must name one of the endpoint's own source_ids. -ep-ansible-managed: - name: "Managed Hosts Only" - source_ids: ["src-section9", "src-infra"] - output: ansible - limit: - by_hosts_from_inventory: "src-section9" - -# Script transformer — for a bespoke format. -ep-dns-zone: - name: "DNS zone file" - source_ids: ["src-infra"] - script_path: "outputs/dns_zone.py" - project_id: "prj-connectors" # optional: resolve inside a checkout - script_args: [] # optional CLI args for the script - timeout_seconds: 300 # script only: abort a transform that takes longer (default 300) +# endpoints/ep-ansible-full.yaml — builtin transformer, no process spawn +endpoints: + - name: ep-ansible-full + description: "Full Ansible Inventory" + sources: ["src-section9", "src-infra"] + output: ansible # builtins: ansible, json, csv + config: + filter: "datacenter=section9" # host, group, filter, fields, exclude_fields (see docs/endpoints.md) + +# endpoints/ep-ansible-managed.yaml — a constructed inventory: merge everything, +# return only the hosts one source has. The others still enrich them and still +# group them. Optional; a limit must name one of the endpoint's own sources. +endpoints: + - name: ep-ansible-managed + description: "Managed Hosts Only" + sources: ["src-section9", "src-infra"] + output: ansible + limit: + by_hosts_from_inventory: "src-section9" + +# endpoints/ep-dns-zone.yaml — script transformer, for a bespoke format +endpoints: + - name: ep-dns-zone + description: "DNS zone file" + sources: ["src-infra"] + project: "prj-connectors" # optional: resolve script_path in a checkout + script_path: "outputs/dns_zone.py" + script_args: [] # optional CLI args for the script + timeout_seconds: 300 # script only: abort a transform that takes longer (default 300) ``` --- -## projects.yaml +## projects -Git repositories that hold connector/enricher/transformer scripts — the field -reference; the full lifecycle (boot behavior, script resolution, virtualenvs, -health) is [its own page](projects.md). At boot the -app clones each project (shallow, one directory per project id under the +Git repositories that hold connector/enricher/transformer scripts and static +inventories — the field reference; the full lifecycle (boot behavior, script +resolution, virtualenvs, health) is [its own page](projects.md). At boot the +app clones each project (shallow, one directory per project name under the `projects.dir` from `config.yaml`, default `./projects`) and re-pulls it every `sync_interval_seconds` (fetch + hard reset: local drift is discarded). The boot clones run in the **background, after the listener is up**: they run @@ -346,16 +569,18 @@ Kubernetes startup probe. The sync schedulers start once the clones have had their bounded chance. ```yaml -prj-connectors-infra: - name: "Infrastructure Connectors" - git_url: "https://github.com/OpusProjects/connectors-infra.git" - branch: "main" # default "main" - credential_id: "cred-github-token" # optional, for private repos - sync_interval_seconds: 1800 # 0/absent = no periodic re-pull - # schedule: "15 2 * * *" # OR a cron cadence (UTC) — pick one - timeout_seconds: 300 # abort a clone/pull that runs longer (default 300) - python_venv: false # build a venv from requirements.txt; see below - sync_on_boot: true # default true; see below +# projects/prj-connectors-infra.yaml +projects: + - name: prj-connectors-infra + description: "Infrastructure Connectors" + git_url: "https://github.com/OpusProjects/connectors-infra.git" + branch: "main" # default "main" + credential: "cred-github-token" # optional, for private repos + sync_interval_seconds: 1800 # 0/absent = no periodic re-pull + # schedule: "15 2 * * *" # OR a cron cadence (UTC) — pick one + timeout_seconds: 300 # abort a clone/pull that runs longer (default 300) + python_venv: false # build a venv from requirements.txt; see below + sync_on_boot: true # default true; see below ``` Three sync styles compose from those two knobs: @@ -374,16 +599,16 @@ the scripts repo can push and then call the sync endpoint (admin key) to roll new scripts without restarting; script files are re-read on every execution, so the update takes effect on the next run. -How script paths resolve: a *relative* `script_path` whose file exists inside -the checkout runs from there (`//`); -otherwise the configured path is kept as-is (image-baked and mounted scripts -keep working, with a warning when the checkout exists but the file doesn't). -Resolution happens at **every execution**, not once at boot, so a script that -first appears after startup — a slow clone, a pipeline's first push — is used -on the very next run without a restart. -SSH sources are never resolved — their `script_path` is a remote command. -Sources always declare `project_id`; enrichers and endpoints may add an -optional `project_id` to resolve their scripts the same way. Scripts must +How script paths resolve: a *relative* path whose file exists inside the +checkout runs from there (`//`); otherwise the +configured path is kept as-is (image-baked and mounted scripts keep working, +with a warning when the checkout exists but the file doesn't). Resolution +happens at **every execution**, not once at boot, so a script that first +appears after startup — a slow clone, a pipeline's first push — is used on the +very next run without a restart. The project comes from the connector (its +`project` + `path`), from a static source (its own `project` + `path`), or from +an enricher's or endpoint's optional `project` (for their `script_path`). SSH +connectors are never resolved — their `path` is a remote command. Scripts must carry the executable bit in git (`git update-index --chmod=+x`). With `python_venv: true`, a `requirements.txt` in the checkout gets a real @@ -406,35 +631,272 @@ at sync time and the periodic re-pull retries. --- -## api_keys.yaml +## api_keys API keys with per-consumer permissions. The secret is NEVER here — `env` names -the environment variable that holds it, so this file can live in git and +the environment variable that holds it, so these files can live in git and rotation stays an external process (swap the env var, restart). ```yaml -key-awx: - name: "AWX" - env: "UNIFIED_API_KEY_AWX" - role: "admin" # everything - -key-forms: - name: "AnsibleForms" - env: "UNIFIED_API_KEY_FORMS" - # role defaults to "restricted": only what is listed below - sources: ["src-ssh-linux"] # may list/read/sync these sources (or views) - endpoints: ["ep-ansible-full"] # may list/run these endpoints +# api_keys/consumers.yaml +api_keys: + - name: key-awx + description: "AWX" + env: "UNIFIED_API_KEY_AWX" + role: "admin" # everything + + - name: key-forms + description: "AnsibleForms" + env: "UNIFIED_API_KEY_FORMS" + # role defaults to "restricted": only what is listed below + sources: ["src-ssh-linux"] # may list/read/sync these sources (or views) + endpoints: ["ep-ansible-full"] # may list/run these endpoints ``` A declared key whose env var is missing or empty fails startup (a typo must not silently lock a consumer out at request time). Restricted keys referencing unknown source/endpoint ids also fail startup; a `sources:` entry may name a -[view](views.md), which grants the view without granting its members. No file and no -`UNIFIED_API_KEY` = open API (with a loud warning). See +[view](views.md), which grants the view without granting its members. No +`api_keys` items and no `UNIFIED_API_KEY` = open API (with a loud warning). See [API → Authentication](api.md#authentication) for the exact route semantics. --- +## Upgrading from 0.34 + +Until 0.35 the directory was eight fixed files — `config.yaml` plus +`credentials.yaml`, `projects.yaml`, `sources.yaml`, `views.yaml`, +`enrichers.yaml`, `endpoints.yaml` and `api_keys.yaml` — each a map keyed by id, +and every source carried its own script. **0.35 converts that layout itself**, +so a pod upgraded in place starts instead of refusing its own volume. + +### At startup + +A `CONFIG_DIR` whose root still holds one of those legacy file names, with a map +of maps at its top level, is migrated in place **once**, before loading: + +1. **Plan, in memory.** Every item becomes `
/.yaml`: the map key + becomes `name`, the old `name` becomes `description`, the reference keys + lose their `_id`. Values are written as they were: `1.10`, `0x1F` and + `True` stay exactly that, never rewritten as the number they look like. +2. **Extract connectors.** Sources sharing the same type, project, script, + `script_args`, `host_args` and `output_format` share one connector, named + after the script (its folder when the file is generic — `d42/inventory.py` + becomes `d42`), `ssh-` for ssh. A name already taken (by another + connector, or by a built-in: a script in a `remote/` folder) gets the lowest + free `-2`, `-3`, ... suffix instead. The + timeout most of its sources used becomes the connector's; a source that + differed keeps its own as an override. ssh sources lose the `project_id` they + were required to set and never used; a remote source's `script_path` moves to + `config.source`, a static source's to `path`. +3. **Validate.** The plan is written into a hidden staging folder beside the + real files and loaded exactly as startup would load it. +4. **Commit.** The old files move to `.legacy-config-/` (hidden, + so the loader skips it) and the new ones are renamed into place. + +Every step is logged at WARN — which files, which connector came from which +sources, what was dropped — because an operator should find out from the log, +not from a diff weeks later. A conversion that would not load **refuses to +start, having changed nothing**: the old files stay where they were, and the +error says why. + +Comments in the old files are not carried over: they are not data, and a YAML +reader does not see them. They are still in the backup, next to the new files, +for whoever wants to move them across. + +**Commit the new layout wherever the directory comes from.** A pipeline that +pushes the old files back over a migrated directory is pushing a layout this +version no longer reads, and the config API will say so (with a hint naming +`--migrate-config`). + +### A read-only directory + +A ConfigMap or a `:ro` bind mount cannot be converted in place. The conversion +is then written into a **private temporary copy**, which is what the process +loads, with a loud warning. Nothing is lost by that — the source of truth is +wherever the read-only files come from — but every restart converts them again, +and writes through the [config API](config-api.md) land in the copy. Convert +the source with `--migrate-config` and redeploy. The copy is removed when the +process stops; one left behind by a process that was killed is removed by the +next start (a copy in use is locked, so instances sharing a temp folder never +remove each other's). + +### In a configuration repository + +```bash +unified-api --migrate-config ./config # convert in place, print what it did, exit +CONFIG_DIR=./config unified-api --check-config # on a 0.34 directory: validate the + # conversion, write nothing +``` + +`--migrate-config` runs the same code as startup, so a repository and a pod +upgraded in place end up with the same files. See [CLI](cli.md#converting-a-034-directory---migrate-config). + +### Rolling back + +0.34 cannot read item files, so rolling back needs the old files back: move +them from `.legacy-config-/` to the root. 0.34 reads only its eight +file names, so the new folders are ignored rather than refused — but any change +made since the upgrade lives only in them, and has to be carried back by hand. + +### Before and after + +A script source (two tenants of the same system become one connector): + +```yaml +# 0.34 — sources.yaml +src-d42-a: + name: "Device42 tenant A" + connector_type: "script" + project_id: "prj-inventory" + script_path: "dynamic/d42/inventory.py" + script_args: ["--list"] + output_format: "ansible" + credential_ids: ["cred-d42"] + sync_interval_seconds: 300 + ttl_seconds: 3600 + config: + tenant: "a" +src-d42-b: + # ... the same, with tenant "b" +``` + +```yaml +# 0.35 — connectors/d42.yaml +connectors: + - name: d42 + type: "script" + project: "prj-inventory" + path: "dynamic/d42/inventory.py" + script_args: ["--list"] + output_format: "ansible" + +# 0.35 — sources/src-d42-a.yaml (src-d42-b.yaml alike) +sources: + - name: src-d42-a + description: "Device42 tenant A" + connector: "d42" + credentials: ["cred-d42"] + sync_interval_seconds: 300 + ttl_seconds: 3600 + config: + tenant: "a" +``` + +An ssh source (the unused `project_id` is dropped): + +```yaml +# 0.34 +src-ssh-dc1: + name: "Facts DC1" + connector_type: "ssh" + project_id: "prj-inventory" + script_path: "gather_facts" + credential_ids: ["cred-ssh"] + sync_interval_seconds: 86400 + ttl_seconds: 3600 + hosts_from_source: + source: "src-d42-a" + config: + gather_mode: "facts" +``` + +```yaml +# 0.35 — connectors/ssh-gather-facts.yaml +connectors: + - name: ssh-gather-facts + type: "ssh" + path: "gather_facts" + +# 0.35 — sources/src-ssh-dc1.yaml +sources: + - name: src-ssh-dc1 + description: "Facts DC1" + connector: "ssh-gather-facts" + credentials: ["cred-ssh"] + sync_interval_seconds: 86400 + ttl_seconds: 3600 + hosts_from_source: + source: "src-d42-a" + config: + gather_mode: "facts" +``` + +A remote source (the remote source id moves into `config`): + +```yaml +# 0.34 +src-edge-dc2: + name: "Datacenter 2" + connector_type: "remote" + project_id: "prj-unused" + script_path: "src-ssh" + credential_ids: ["cred-edge-dc2"] + sync_interval_seconds: 120 + ttl_seconds: 600 + config: + url: "https://unified-api-dc2.example.com" +``` + +```yaml +# 0.35 — sources/src-edge-dc2.yaml +sources: + - name: src-edge-dc2 + description: "Datacenter 2" + connector: "remote" + credentials: ["cred-edge-dc2"] + sync_interval_seconds: 120 + ttl_seconds: 600 + config: + source: "src-ssh" + url: "https://unified-api-dc2.example.com" +``` + +A static inventory source (`script_path` becomes `path`, on the source): + +```yaml +# 0.34 +src-linux: + name: "Linux static inventory" + connector_type: "static_inventory" + project_id: "prj-inventories" + script_path: "linux/inventory.yaml" + sync_interval_seconds: 300 + ttl_seconds: 600 +``` + +```yaml +# 0.35 — sources/src-linux.yaml +sources: + - name: src-linux + description: "Linux static inventory" + connector: "static_inventory" + project: "prj-inventories" + path: "linux/inventory.yaml" + sync_interval_seconds: 300 + ttl_seconds: 600 +``` + +The renamed reference keys, for anything converted by hand: + +| Item | 0.34 | 0.35 | +|---|---|---| +| source | `credential_ids` | `credentials` | +| source | `connector_type`, `project_id`, `script_path`, `script_args`, `host_args`, `output_format` | `connector` (the rest moved to the connector item) | +| project | `credential_id` | `credential` | +| enricher | `target_id`, `source_id`, `project_id` | `target`, `source`, `project` | +| endpoint | `source_ids`, `project_id` | `sources`, `project` | +| every item | the map key / `name` | `name` / `description` | + +Unchanged: a view member's `source` and `owns.source`, `hosts_from_source.source`, +an endpoint's `limit.by_hosts_from_inventory`, an API key's `sources` / +`endpoints`, an enricher's and endpoint's `script_path`, and LDAP's +`bind_credential` in `config.yaml`. The API is unchanged too: responses keep +`source_id`, `name`, `target_id`, `source_ids` and the rest, so no consumer +needs to change. + +--- + ## Environment variables Everything the process reads from the environment, including where each credential looks for its secret. @@ -442,7 +904,7 @@ Everything the process reads from the environment, including where each credenti | Variable | Effect | |---|---| | `CONFIG_DIR` | Config directory (default `config`) | -| `UNIFIED_API_KEY` | Legacy single admin key: when set, it authenticates like an `api_keys.yaml` admin entry (see [API](api.md)); health probes and Swagger stay public | -| per `api_keys.yaml` | Each key definition names the env var holding its secret | +| `UNIFIED_API_KEY` | Legacy single admin key: when set, it authenticates like an admin `api_keys` item (see [API](api.md)); health probes and Swagger stay public | +| per `api_keys` item | Each key definition names the env var holding its secret | | `RUST_LOG` | Log filter, e.g. `debug` or `unified_api=debug` (default `info`) | -| `_` | Secret values referenced by `credentials.yaml` `env_prefix`/`secret_keys` | +| `_` | Secret values referenced by a credential's `env_prefix`/`secret_keys` | diff --git a/docs/connectors.md b/docs/connectors.md index 35030b0..1bf3f7a 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -2,13 +2,15 @@ Everything pluggable in Unified API is an **executable** speaking JSON. Any language works — the shipped examples under `tests/` are Python. This page defines the -three contracts. +three contracts, and the connector item that says how a source reaches its system. - [The Dataset shape](#the-dataset-shape) -- [Source connectors (`connector_type: script`)](#source-connectors-connector_type-script) -- [Source connectors (`connector_type: ssh`)](#source-connectors-connector_type-ssh) -- [Static inventory sources (`connector_type: static_inventory`)](#static-inventory-sources-connector_type-static_inventory) -- [Remote sources — federation (`connector_type: remote`)](#remote-sources--federation-connector_type-remote) +- [Connectors and sources](#connectors-and-sources) +- [Script connectors (`type: script`)](#script-connectors-type-script) +- [SSH connectors (`type: ssh`)](#ssh-connectors-type-ssh) +- [Static inventory sources (`connector: static_inventory`)](#static-inventory-sources-connector-static_inventory) +- [Remote sources — federation (`connector: remote`)](#remote-sources--federation-connector-remote) +- [Inspecting connectors (`/api/v1/connectors`)](#inspecting-connectors-apiv1connectors) - [Enrichers](#enrichers) - [Output endpoints](#output-endpoints) - [Testing your script](#testing-your-script) @@ -40,20 +42,91 @@ only meaningful in enricher output. --- -## Source connectors (`connector_type: script`) +## Connectors and sources + +A **connector** says HOW to talk to a kind of system: which script, from which +project, with which arguments, and how to read what it prints. A **source** says +WHICH instance of that system — its credentials, its `config`, its schedule and +TTLs — and names its connector. Ten Device42 tenants are ten sources and one +connector; before connectors existed, the script path, its arguments and its +output format were copied into each of the ten, and drifted apart one edit at a +time. + +```yaml +# connectors/d42.yaml +connectors: + - name: d42 + description: "Device42 inventory script" + type: "script" # script (default) | ssh + project: "prj-inventory" # the checkout `path` resolves in + path: "dynamic/d42/inventory.py" + script_args: ["--list"] + output_format: "ansible" + # timeout_seconds: 300 # default; a source may override it + +# sources/src-d42-acme.yaml +sources: + - name: src-d42-acme + description: "Device42 Acme" + connector: "d42" + credentials: ["cred-d42-acme"] + sync_interval_seconds: 300 + ttl_seconds: 3600 + config: + tenant: "acme" +``` + +| Connector field | Meaning | +|---|---| +| `type` | `script` (default) or `ssh` | +| `project` | The project whose checkout holds the script. Required for a script connector whose `path` is relative; refused on ssh, whose `path` runs on the remote host | +| `path` | script: the file to run, inside the project checkout (or absolute). ssh: the remote command for `gather_mode: script`, or a label such as `gather_facts` for the built-in gather modes | +| `script_args` | CLI arguments, verbatim. On ssh, appended to the remote command in `script` gather mode | +| `host_args` | Script only: CLI arguments for a host-scoped sync, replacing `script_args` (see below) | +| `output_format` | Script only: `native` (default) or `ansible` (see below) | +| `timeout_seconds` | Bound on one sync (default 300). A source may set its own to override it — the tenant ten times bigger than the others | + +Anything that describes the script or the protocol is the connector's; anything +about one instance — credentials, `config`, cadence, TTLs, and cache policy such +as `sync_mode` and `host_sync_updates_group_vars` — stays on the source. The +full split is in [configuration](configuration.md#source-or-connector). + +Settings that look active and are not are refused at startup rather than +ignored: `project`, `host_args` or `output_format` on an ssh connector, a +relative script `path` without a `project`, and `project` / `path` on any source +that is not `static_inventory`. + +Two connectors are **built in** and need no item, because there is nothing about +their HOW to configure — the protocol is fixed, and the WHERE is per source: + +| `connector:` | What it reads | The source sets | +|---|---|---| +| `remote` | Another unified-api instance | `config.url` and `config.source` (the source id there) | +| `static_inventory` | An Ansible YAML inventory in a project checkout | `project` and `path` | + +Their names are reserved: a connector item called `remote` or `static_inventory` +fails startup. `project` + `path` mean the same thing everywhere — a file inside +that project's checkout — whether it is a connector's script or a static source's +inventory. + +--- + +## Script connectors (`type: script`) The connector script is executed on every sync and must print a Dataset to stdout. -**Input (command line):** the source's `script_args` list is passed verbatim as +**Input (command line):** the connector's `script_args` list is passed verbatim as CLI arguments (no shell involved, so no quoting concerns). This is how scripts that implement the standard Ansible dynamic inventory interface get their `--list`: ```yaml -src-d42: - script_path: "d42_inventory.py" - script_args: ["--list"] - output_format: "ansible" # see below — such scripts emit Ansible JSON +connectors: + - name: d42 + project: "prj-inventory" + path: "d42_inventory.py" + script_args: ["--list"] + output_format: "ansible" # see below — such scripts emit Ansible JSON ``` Without `script_args` the script is invoked bare, exactly as before. @@ -75,8 +148,8 @@ Anything else a script needs must travel in its `config` map or a credential. **Output:** the full Dataset JSON on stdout. Exit non-zero to fail the sync; stderr is captured into the error message. -**Time limit:** the script must finish within the source's `timeout_seconds` -(default 300). A slower run is aborted and the sync fails with +**Time limit:** the script must finish within the source's `timeout_seconds`, or +the connector's when the source sets none (default 300). A slower run is aborted and the sync fails with `sync timed out after Ns` — a hung script never blocks the scheduler or an API call. The process is **killed**, not merely abandoned, so a script that wedges does not accumulate a live copy per sync interval. Write scripts to be interruptible at any @@ -103,14 +176,16 @@ single host or group without paying for a full inventory pull. `scope`/`target` reach the script inside `SOURCE_CONFIG`, which is a convention of ours. Inventory scripts have their own: Ansible defines `--list` and -`--host `. A source can name the arguments a **host-scoped** sync +`--host `. A connector can name the arguments a **host-scoped** sync should use, and they replace `script_args` for that run: ```yaml -src-d42: - script_path: "connectors/d42_inventory.py" - script_args: ["--list"] - host_args: ["--host-with-groups", "{target}"] +connectors: + - name: d42 + project: "prj-inventory" + path: "connectors/d42_inventory.py" + script_args: ["--list"] + host_args: ["--host-with-groups", "{target}"] ``` `{target}` is substituted with the requested hostnames, comma-joined — the same @@ -130,8 +205,8 @@ per-host calls vestigial, Ansible's own `--host` was usually implemented as "fetch everything, then filter" — but a script that CAN fetch one host should be asked for one host. -`host_args` is refused at startup on any source that is not a script source: the -other connectors are not spawned with CLI arguments at all. +`host_args` is refused at startup on an ssh connector, and the built-ins have no +place for it: none of them is spawned with CLI arguments at all. #### What a host-scoped sync writes @@ -188,11 +263,18 @@ print(json.dumps({"hostvars": inventory.hostvars, "groups": inventory.groups})) ``` ```yaml -src-d42: - script_path: "connectors/d42_inventory.py" - script_args: ["--list"] - host_args: ["--host-with-groups", "{target}"] - host_sync_updates_group_vars: true +connectors: + - name: d42 + project: "prj-inventory" + path: "connectors/d42_inventory.py" + script_args: ["--list"] + host_args: ["--host-with-groups", "{target}"] + +sources: + - name: src-d42 + connector: "d42" + host_sync_updates_group_vars: true # cache policy: the source's, not the script's + ttl_seconds: 3600 ``` The pay-off is in what the granular call does NOT do. Registering a new machine @@ -207,15 +289,17 @@ script. Scripts written for Ansible print a different JSON shape than the Dataset: hostvars under `_meta.hostvars` and groups as top-level keys. With -`output_format: "ansible"` on the source, that output is converted to a +`output_format: "ansible"` on the connector, that output is converted to a Dataset on the fly — any existing dynamic inventory script works unmodified (pair it with `script_args: ["--list"]`): ```yaml -src-d42: - script_path: "d42_inventory.py" - script_args: ["--list"] - output_format: "ansible" +connectors: + - name: d42 + project: "prj-inventory" + path: "d42_inventory.py" + script_args: ["--list"] + output_format: "ansible" ``` Conversion rules: @@ -231,7 +315,7 @@ Conversion rules: - Malformed input is an **error that fails the sync**, naming the offending group — never a silent skip. -**Misconfiguration safety net:** if a source left on the default +**Misconfiguration safety net:** if a connector left on the default `output_format: native` parses to 0 hosts and 0 groups but the output contains `_meta`, the sync logs a WARN suggesting `output_format: "ansible"`. (Both Dataset fields are optional in JSON, so Ansible output "parses fine" as an @@ -239,10 +323,35 @@ empty inventory — that silent zero is the failure mode this flag exists for.) --- -## Source connectors (`connector_type: ssh`) +## SSH connectors (`type: ssh`) The native SSH connector needs no script on the API host — it connects to the fleet -in parallel and builds the Dataset from what it finds. +in parallel and builds the Dataset from what it finds. The connector item holds +only what runs remotely; which hosts, as whom, and how is the source's: + +```yaml +# connectors/ssh-facts.yaml +connectors: + - name: ssh-facts + description: "SSH fact gathering" + type: "ssh" + path: "gather_facts" # a label in facts mode; the remote command in script mode + +# sources/src-ssh-dc1.yaml +sources: + - name: src-ssh-dc1 + connector: "ssh-facts" + credentials: ["cred-ssh-infra"] + sync_interval_seconds: 300 + ttl_seconds: 600 + config: + hosts: "host1.example.com,host2.example.com" + gather_mode: "facts" +``` + +`path` is never resolved in a checkout — nothing runs locally — which is why an +ssh connector refuses `project`, and refuses `host_args` and `output_format` too: +it builds its own command per host and its own dataset. **Source `config` keys:** @@ -252,7 +361,7 @@ in parallel and builds the Dataset from what it finds. | `port` | `22` | SSH port | | `concurrency` | `50` | Max parallel connections (tokio semaphore) | | `ssh_connect_timeout_seconds` | `30` | Per-host connection/exec timeout | -| `gather_mode` | `facts` | `facts` reads Ansible local facts; `script` runs `script_path` remotely | +| `gather_mode` | `facts` | `facts` reads Ansible local facts; `script` runs the connector's `path` remotely | | `fact_path` | `/etc/ansible/facts.d` | Where facts live (facts mode) | | `ssh_legacy_algorithms` | `false` | `"true"` additionally offers SHA-1 KEX/MACs (appended last, modern servers still pick modern ones) for OpenSSH 5.x-era hosts (EL6) that lack `hmac-sha2` | | `ssh_known_hosts` | — | Path to an OpenSSH `known_hosts` file. When set, every server key is checked against it **before authentication**: an unknown or mismatched host is refused (it lands in `unreachable`, so its last known data is kept), and the refusal logs both fingerprints. A server presenting an OpenSSH host *certificate* is refused too — verification is defined over plain host keys. Unset = any server key is accepted, and every sync warns about it | @@ -275,9 +384,9 @@ without negotiation, the same key "mysteriously" works with the OpenSSH client but not through the API. ed25519/ecdsa keys are unaffected (no hash to negotiate). -In `script` mode, `script_args` are appended to the remote command -(`script_path arg1 arg2 ...`); in `facts` mode they are ignored (the remote -command is fixed). +In `script` mode, the connector's `script_args` are appended to the remote command +(`path arg1 arg2 ...`); in `facts` mode they are ignored (the remote command is +fixed). ### Dynamic host lists (`hosts_from_source`) @@ -286,19 +395,20 @@ Instead of a static `config.hosts`, an SSH source can take its hosts from source says WHAT exists, SSH says HOW it is doing": ```yaml -src-fleet-facts: - connector_type: "ssh" - credential_ids: ["cred-fleet-ssh"] - sync_interval_seconds: 300 - ttl_seconds: 600 - hosts_from_source: - source: "src-netbox" # any source: script, static, even ssh - match_pattern: # absent = every host in the dataset - groups: ["linux", "proxmox_vms"] - hosts: ["extra01.example.com"] - connect_via: "ansible_host_then_hostname" - config: - gather_mode: "facts" +sources: + - name: src-fleet-facts + connector: "ssh-facts" + credentials: ["cred-fleet-ssh"] + sync_interval_seconds: 300 + ttl_seconds: 600 + hosts_from_source: + source: "src-netbox" # any source: script, static, even ssh + match_pattern: # absent = every host in the dataset + groups: ["linux", "proxmox_vms"] + hosts: ["extra01.example.com"] + connect_via: "ansible_host_then_hostname" + config: + gather_mode: "facts" ``` Semantics: @@ -331,15 +441,15 @@ others — it just occupies one of the `concurrency` slots until its timeout (up to 2× `ssh_connect_timeout_seconds` with a fallback strategy). > `ssh_connect_timeout_seconds` bounds a **single host** connection; the -> source-level `timeout_seconds` (default 300) separately bounds the **whole -> sync** across all hosts. They are different knobs. +> connector's `timeout_seconds` (default 300, overridable per source) separately +> bounds the **whole sync** across all hosts. They are different knobs. **Credentials:** expects `username` (or `ssh_username`) and an `ssh_key_path` / `key_path` from `file_keys` — see [configuration](configuration.md). --- -## Static inventory sources (`connector_type: static_inventory`) +## Static inventory sources (`connector: static_inventory`) For classic Ansible **static YAML inventories** — an `inventory.yaml` with the `all/children/hosts` tree plus optional `group_vars/` and `host_vars/` @@ -358,20 +468,22 @@ accumulate, and a host inherits the vars of every ancestry its group is declared under. Naming the same host in both declarations lists it once. ```yaml -src-inventory-linux: - name: "Linux static inventory" - connector_type: "static_inventory" - project_id: "prj-inventories" # git repo holding the inventory - script_path: "inventory.yaml" # path to the file inside the checkout - sync_interval_seconds: 300 - ttl_seconds: 600 +sources: + - name: src-inventory-linux + description: "Linux static inventory" + connector: "static_inventory" # built in: no connector item + project: "prj-inventories" # git repo holding the inventory + path: "inventory.yaml" # the file inside the checkout + sync_interval_seconds: 300 + ttl_seconds: 600 ``` -`script_path` doubles as "path to the inventory file"; with a git project it -resolves inside the checkout, so the project's periodic pull (or the -on-demand `POST /api/v1/projects/{id}/sync`) is what brings in new data — the -next source sync reads the updated files. `script_args`, `output_format`, -credentials and `SOURCE_CONFIG` don't apply to this connector. +A built-in connector, so the source names the file itself: `project` + `path`, the +same pair a script connector uses for its script. Both are required. The path +resolves inside the checkout, so the project's periodic pull (or the on-demand +`POST /api/v1/projects/{id}/sync`) is what brings in new data — the next source +sync reads the updated files. Arguments, output formats, credentials and +`SOURCE_CONFIG` don't apply to this connector. **What lands where:** @@ -413,7 +525,7 @@ value whose type matters. --- -## Remote sources — federation (`connector_type: remote`) +## Remote sources — federation (`connector: remote`) Another unified-api instance is the source. This is how multi-datacenter setups compose: one instance per DC does the local work (SSH with LAN @@ -421,29 +533,33 @@ latency, local credentials, local firewall rules), and a central instance aggregates them — consumers only ever talk to the central. ```yaml -# on the central -src-dc1: - name: "Datacenter A" - connector_type: "remote" - project_id: "prj-unused" # required by schema; unused here - script_path: "src-ssh" # the source id ON THE REMOTE instance - credential_ids: ["cred-edge-dc1"] # token credential = the remote API key - sync_interval_seconds: 120 - ttl_seconds: 600 - config: - url: "https://unified-api-dc1.example.com" - # http_timeout_seconds: "30" # default 30 - # insecure_tls: "true" # self-signed remotes; opt-in, never default +# on the central: sources/src-dc1.yaml +sources: + - name: src-dc1 + description: "Datacenter A" + connector: "remote" # built in: no connector item + credentials: ["cred-edge-dc1"] # token credential = the remote API key + sync_interval_seconds: 120 + ttl_seconds: 600 + config: + url: "https://unified-api-dc1.example.com" + source: "src-ssh" # the source id ON THE REMOTE instance + # http_timeout_seconds: "30" # default 30 + # insecure_tls: "true" # self-signed remotes; opt-in, never default ``` +Both `config.url` and `config.source` are required, and `project` / `path` are +refused: nothing runs from a checkout. + On the edge, give the central a **restricted API key** (least privilege): ```yaml -# api_keys.yaml on the edge -key-central: - name: "Central aggregator" - env: "UNIFIED_API_KEY_CENTRAL" - sources: ["src-ssh"] +# on the edge: api_keys/key-central.yaml +api_keys: + - name: key-central + description: "Central aggregator" + env: "UNIFIED_API_KEY_CENTRAL" + sources: ["src-ssh"] ``` **How it works:** `GET /dataset` on the remote returns exactly the Dataset a @@ -477,6 +593,48 @@ sources to serve one merged world view. --- +## Inspecting connectors (`/api/v1/connectors`) + +Which connectors exist, what each one runs, which sources use it and how those +sources are doing — admin-only, like projects, since a connector names the +script, its project and its arguments for every source of a system: + +```bash +curl -s localhost:8182/api/v1/connectors -H "X-API-Key: $ADMIN_KEY" +curl -s localhost:8182/api/v1/connectors/d42 -H "X-API-Key: $ADMIN_KEY" +``` + +```json +{ + "connector_id": "d42", + "name": "Device42 inventory script", + "connector_type": "script", + "built_in": false, + "project_id": "prj-inventory", + "path": "dynamic/d42/inventory.py", + "script_args": ["--list"], + "host_args": [], + "output_format": "ansible", + "timeout_seconds": 300, + "used_by": [ + {"source_id": "src-d42-a", "name": "Device42 tenant A", "timeout_seconds": 300, "failing": false, "synced": true}, + {"source_id": "src-d42-b", "name": "Device42 tenant B", "timeout_seconds": 900, "failing": true, "synced": true} + ], + "health": {"sources": 2, "healthy": 1, "failing": 1, "never_synced": 0} +} +``` + +The list holds the connector items first, then the two built-ins +(`built_in: true`, `connector_type` `remote` / `static_inventory`, null +`project_id`, `path`, `output_format` and `timeout_seconds`), so every source's +`connector:` resolves to a row. `used_by[].timeout_seconds` is the bound that +source actually syncs under — its override, or the connector's — and `health` +counts its sources' sync health at a glance: one broken script shows up as a +connector whose sources all fail at once. `output_format` is null for ssh. +`GET /api/v1/sources/{id}` names the connector under `gathering.connector`. + +--- + ## Enrichers An enricher post-processes a dataset already in the cache: resolve DNS, probe @@ -508,7 +666,7 @@ script is running are not lost (the enricher only overwrites hosts it returns). > **Return the host's full variable map, not only your own keys.** A script > enricher's `hostvars` entry *replaces* the host's map rather than patching it key > by key, so any key you omit is dropped. The dataset arrives on stdin precisely so -> you can carry the rest through. (A *declarative* enricher — `source_id` + +> you can carry the rest through. (A *declarative* enricher — `source` + > `fields` — has no such constraint: it writes only the fields it owns.) --- @@ -541,8 +699,9 @@ clear error when exceeded. ## Testing your script -Wire it into `config/sources.yaml` (or enrichers/endpoints) pointing at your file, -`cargo run`, then drive it through the API: +Wire it into a connector item pointing at your file (`config/connectors/mine.yaml`) +and a source naming it (`config/sources/src-mine.yaml`) — or an enricher or +endpoint item — then `cargo run` and drive it through the API: ```bash curl -X POST localhost:8182/api/v1/sources/src-mine/sync diff --git a/docs/deployment.md b/docs/deployment.md index 61104a1..dfae8c3 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -9,6 +9,7 @@ and what changes when the target is Kubernetes, ArgoCD or a CI pipeline. - [CI/CD pipeline](#cicd-pipeline) - [Kubernetes notes](#kubernetes-notes) - [GitOps with ArgoCD](#gitops-with-argocd) +- [Upgrading from 0.34](#upgrading-from-034) - [Scheduling and observability](#scheduling-and-observability) --- @@ -26,7 +27,7 @@ imported libraries preinstalled from apt — `requests`, `PyYAML`, `jinja2` — and `git` for project checkouts. Scripts needing anything beyond that declare it in their project's `requirements.txt` and set `python_venv: true` on the project — the app builds and maintains a virtualenv per project (see -[configuration](configuration.md#projectsyaml)); a derived image is only +[configuration](configuration.md#projects)); a derived image is only needed for non-Python tooling. ```bash @@ -46,9 +47,26 @@ complete example below. One `config/` directory exercising **every connector type, every credential type and every secret-delivery mechanism**. Copy it, delete what you don't -need. The pieces reference each other by id, so the startup validation will +need. The pieces reference each other by name, so the startup validation will tell you if you break a reference while trimming. +``` +config/ +├── config.yaml +├── credentials/ cred-d42.yaml cred-netbox.yaml cred-fleet-ssh.yaml cred-gitlab.yaml +├── projects/ prj-connectors.yaml prj-inventories.yaml +├── connectors/ d42.yaml netbox.yaml ssh-facts.yaml ssh-disk-report.yaml +├── sources/ src-d42.yaml src-netbox.yaml src-fleet-facts.yaml src-fleet-disks.yaml src-inventory.yaml +├── enrichers/ enrich-reachability.yaml +├── endpoints/ ep-awx-full.yaml +└── api_keys/ key-awx.yaml key-forms.yaml +``` + +One item per file, as below; the folders are a convention for people, and the +top-level key in each file is what says what it holds (see +[configuration](configuration.md#the-directory)). The blocks below show each +folder's files one after another. + ### config.yaml — server, cache persistence, project checkouts ```yaml @@ -68,7 +86,7 @@ projects: dir: "/var/lib/unified-api/projects" ``` -### credentials.yaml — all three types, both delivery mechanisms +### credentials — all three types, both delivery mechanisms Credentials never hold secrets; they say **where to read them**. Two mechanisms: environment variables (`env_prefix` + `secret_keys`) or files @@ -76,179 +94,231 @@ mechanisms: environment variables (`env_prefix` + `secret_keys`) or files by path, like SSH keys). ```yaml -# username_password from env vars: reads D42_USERNAME and D42_PASSWORD, -# the connector script receives CREDENTIAL_USERNAME / CREDENTIAL_PASSWORD -cred-d42: - name: "Device42 API" - type: "username_password" - env_prefix: "D42" - secret_keys: - username: "USERNAME" - password: "PASSWORD" - -# token from env: reads NETBOX_TOKEN > script sees CREDENTIAL_TOKEN -cred-netbox: - name: "NetBox API token" - type: "token" - env_prefix: "NETBOX" - secret_keys: - token: "TOKEN" - -# ssh_key: the username from env, the private key as a FILE the connector -# opens by path (file_keys entries are passed as _path) -cred-fleet-ssh: - name: "Fleet SSH" - type: "ssh_key" - env_prefix: "FLEET_SSH" - secret_keys: - username: "USERNAME" - file_keys: - ssh_key: "/run/secrets/fleet-ssh/id_ed25519" - -# token from a JSON file instead of env: {"token": "glpat-..."} — handy when -# the platform delivers secrets as mounted files rather than env vars -cred-gitlab: - name: "GitLab read token" - type: "token" - secret_file: "/run/secrets/gitlab.json" - secret_keys: - token: "token" +# credentials/cred-d42.yaml — username_password from env vars: reads +# D42_USERNAME and D42_PASSWORD, the connector script receives +# CREDENTIAL_USERNAME / CREDENTIAL_PASSWORD +credentials: + - name: cred-d42 + description: "Device42 API" + type: "username_password" + env_prefix: "D42" + secret_keys: + username: "USERNAME" + password: "PASSWORD" + +# credentials/cred-netbox.yaml — token from env: reads NETBOX_TOKEN > script +# sees CREDENTIAL_TOKEN +credentials: + - name: cred-netbox + description: "NetBox API token" + type: "token" + env_prefix: "NETBOX" + secret_keys: + token: "TOKEN" + +# credentials/cred-fleet-ssh.yaml — ssh_key: the username from env, the +# private key as a FILE the connector opens by path (file_keys entries are +# passed as _path) +credentials: + - name: cred-fleet-ssh + description: "Fleet SSH" + type: "ssh_key" + env_prefix: "FLEET_SSH" + secret_keys: + username: "USERNAME" + file_keys: + ssh_key: "/run/secrets/fleet-ssh/id_ed25519" + +# credentials/cred-gitlab.yaml — token from a JSON file instead of env: +# {"token": "glpat-..."} — handy when the platform delivers secrets as +# mounted files rather than env vars +credentials: + - name: cred-gitlab + description: "GitLab read token" + type: "token" + secret_file: "/run/secrets/gitlab.json" + secret_keys: + token: "token" +``` + +### projects — the three sync styles + +```yaml +# projects/prj-connectors.yaml — auto: cloned at boot, re-pulled every 30 min +projects: + - name: prj-connectors + description: "Connector scripts" + git_url: "https://gitlab.example.com/infra/connectors.git" + credential: "cred-gitlab" # private repo > token over https + sync_interval_seconds: 1800 + +# projects/prj-inventories.yaml — manual / pipeline-driven: existing checkout +# used as-is at boot (pair with a persistent volume); updates only via +# POST /api/v1/projects/prj-inventories/sync +projects: + - name: prj-inventories + description: "Static inventories" + git_url: "https://gitlab.example.com/infra/inventories.git" + credential: "cred-gitlab" + sync_on_boot: false ``` -### projects.yaml — the three sync styles +### connectors — how each kind of system is read ```yaml -# Auto: cloned at boot, re-pulled every 30 min -prj-connectors: - name: "Connector scripts" - git_url: "https://gitlab.example.com/infra/connectors.git" - credential_id: "cred-gitlab" # private repo > token over https - sync_interval_seconds: 1800 - -# Manual / pipeline-driven: existing checkout used as-is at boot (pair with a -# persistent volume); updates only via POST /api/v1/projects/prj-inventories/sync -prj-inventories: - name: "Static inventories" - git_url: "https://gitlab.example.com/infra/inventories.git" - credential_id: "cred-gitlab" - sync_on_boot: false +# connectors/d42.yaml — an UNMODIFIED Ansible dynamic inventory script (the +# d42/VMware kind): needs --list and emits _meta-style JSON +connectors: + - name: d42 + description: "Device42 dynamic inventory" + project: "prj-connectors" + path: "d42/d42_inventory.py" # resolved inside the checkout + script_args: ["--list"] + output_format: "ansible" + +# connectors/netbox.yaml — a native-format script (reads SOURCE_CONFIG, +# prints {"hostvars": ..., "groups": ...}) +connectors: + - name: netbox + description: "NetBox" + project: "prj-connectors" + path: "netbox/fetch.py" + +# connectors/ssh-facts.yaml — native SSH reading Ansible local facts: no +# script anywhere, the binary connects in parallel +connectors: + - name: ssh-facts + description: "Fleet facts over SSH" + type: "ssh" + path: "gather_facts" # label in facts mode + +# connectors/ssh-disk-report.yaml — the same SSH connector in script mode: +# run a remote command per host and store its JSON output as that host's vars +connectors: + - name: ssh-disk-report + description: "Disk report over SSH" + type: "ssh" + path: "/usr/local/bin/disk-report" # the remote command + script_args: ["--json"] ``` -### sources.yaml — all three connector types +### sources — every connector type ```yaml -# 1a. script connector running an UNMODIFIED Ansible dynamic inventory script -# (the d42/VMware kind): needs --list and emits _meta-style JSON -src-d42: - name: "Device42" - project_id: "prj-connectors" - script_path: "d42/d42_inventory.py" # resolved inside the checkout - script_args: ["--list"] - output_format: "ansible" - credential_ids: ["cred-d42"] - sync_interval_seconds: 300 - ttl_seconds: 600 - -# 1b. script connector with a native-format script (reads SOURCE_CONFIG, -# prints {"hostvars": ..., "groups": ...}) and per-group TTL overrides -src-netbox: - name: "NetBox" - project_id: "prj-connectors" - script_path: "netbox/fetch.py" - credential_ids: ["cred-netbox"] - sync_interval_seconds: 600 - ttl_seconds: 3600 - ttl_overrides: - groups: - production: 900 - config: # free-form, script-specific - api_url: "https://netbox.example.com" - -# 2. native SSH connector gathering Ansible local facts across a fleet — -# no script anywhere, the binary connects in parallel. The host list is -# DYNAMIC: whatever src-inventory currently knows (chained sources) -src-fleet-facts: - name: "Fleet facts over SSH" - connector_type: "ssh" - project_id: "prj-connectors" # required by schema; unused by ssh - script_path: "gather_facts" # label in facts mode - credential_ids: ["cred-fleet-ssh"] - sync_interval_seconds: 300 - ttl_seconds: 600 - hosts_from_source: - source: "src-inventory" - match_pattern: - groups: ["linux"] # absent = every host in the source - connect_via: "ansible_host_then_hostname" - config: - concurrency: "50" - ssh_connect_timeout_seconds: "30" - gather_mode: "facts" - fact_path: "/etc/ansible/facts.d" - -# 2b. same connector in script mode with a STATIC host list: run a remote -# command per host and store its JSON output as that host's vars -src-fleet-disks: - name: "Fleet disk report" - connector_type: "ssh" - project_id: "prj-connectors" - script_path: "/usr/local/bin/disk-report" - script_args: ["--json"] - credential_ids: ["cred-fleet-ssh"] - ttl_seconds: 3600 # no interval: manual sync only - config: - hosts: "web01.example.com,web02.example.com" - gather_mode: "script" - -# 3. static Ansible YAML inventory parsed natively from a git checkout -# (inventory.yaml + group_vars/ + host_vars/) — no process, no ansible-core -src-inventory: - name: "Static inventory" - connector_type: "static_inventory" - project_id: "prj-inventories" - script_path: "inventory.yaml" - sync_interval_seconds: 300 - ttl_seconds: 600 +# sources/src-d42.yaml +sources: + - name: src-d42 + description: "Device42" + connector: "d42" + credentials: ["cred-d42"] + sync_interval_seconds: 300 + ttl_seconds: 600 + +# sources/src-netbox.yaml — with per-group TTL overrides +sources: + - name: src-netbox + description: "NetBox" + connector: "netbox" + credentials: ["cred-netbox"] + sync_interval_seconds: 600 + ttl_seconds: 3600 + ttl_overrides: + groups: + production: 900 + config: # free-form, script-specific + api_url: "https://netbox.example.com" + +# sources/src-fleet-facts.yaml — the host list is DYNAMIC: whatever +# src-inventory currently knows (chained sources) +sources: + - name: src-fleet-facts + description: "Fleet facts over SSH" + connector: "ssh-facts" + credentials: ["cred-fleet-ssh"] + sync_interval_seconds: 300 + ttl_seconds: 600 + hosts_from_source: + source: "src-inventory" + match_pattern: + groups: ["linux"] # absent = every host in the source + connect_via: "ansible_host_then_hostname" + config: + concurrency: "50" + ssh_connect_timeout_seconds: "30" + gather_mode: "facts" + fact_path: "/etc/ansible/facts.d" + +# sources/src-fleet-disks.yaml — a STATIC host list +sources: + - name: src-fleet-disks + description: "Fleet disk report" + connector: "ssh-disk-report" + credentials: ["cred-fleet-ssh"] + ttl_seconds: 3600 # no interval: manual sync only + config: + hosts: "web01.example.com,web02.example.com" + gather_mode: "script" + +# sources/src-inventory.yaml — static Ansible YAML inventory parsed natively +# from a git checkout (inventory.yaml + group_vars/ + host_vars/) — no +# process, no ansible-core, and no connector item: static_inventory is built in +sources: + - name: src-inventory + description: "Static inventory" + connector: "static_inventory" + project: "prj-inventories" + path: "inventory.yaml" + sync_interval_seconds: 300 + ttl_seconds: 600 ``` -### enrichers.yaml and endpoints.yaml +The one connector type not shown, `remote` (another unified-api instance), is built in +as well; see [federation](federation.md). + +### enrichers and endpoints ```yaml -# enrichers.yaml — post-process a cached source (dataset on stdin, -# partial dataset on stdout) -enrich-reachability: - name: "Probe reachability" - target_id: "src-fleet-facts" - script_path: "enrichers/probe.py" # resolved via project_id if set - project_id: "prj-connectors" - script_args: ["--timeout", "5"] - sync_interval_seconds: 900 +# enrichers/enrich-reachability.yaml — post-process a cached source (dataset +# on stdin, partial dataset on stdout) +enrichers: + - name: enrich-reachability + description: "Probe reachability" + target: "src-fleet-facts" + project: "prj-connectors" + script_path: "enrichers/probe.py" # resolved via project if set + script_args: ["--timeout", "5"] + sync_interval_seconds: 900 ``` ```yaml -# endpoints.yaml — merge cached sources through a transformer script; -# consumers POST here (AWX inventory source, AnsibleForms, ...) -ep-awx-full: - name: "Full AWX inventory" - source_ids: ["src-d42", "src-fleet-facts", "src-inventory"] - project_id: "prj-connectors" - script_path: "outputs/ansible_inventory.py" +# endpoints/ep-awx-full.yaml — merge cached sources through a transformer +# script; consumers POST here (AWX inventory source, AnsibleForms, ...) +endpoints: + - name: ep-awx-full + description: "Full AWX inventory" + sources: ["src-d42", "src-fleet-facts", "src-inventory"] + project: "prj-connectors" + script_path: "outputs/ansible_inventory.py" ``` -### api_keys.yaml — per-consumer permissions +### api_keys — per-consumer permissions ```yaml -key-awx: - name: "AWX" - env: "UNIFIED_API_KEY_AWX" - role: "admin" - -key-forms: - name: "AnsibleForms" - env: "UNIFIED_API_KEY_FORMS" - # restricted (the default): filtered lists, 403 elsewhere - sources: ["src-fleet-facts"] - endpoints: ["ep-awx-full"] +# api_keys/key-awx.yaml +api_keys: + - name: key-awx + description: "AWX" + env: "UNIFIED_API_KEY_AWX" + role: "admin" + +# api_keys/key-forms.yaml +api_keys: + - name: key-forms + description: "AnsibleForms" + env: "UNIFIED_API_KEY_FORMS" + # restricted (the default): filtered lists, 403 elsewhere + sources: ["src-fleet-facts"] + endpoints: ["ep-awx-full"] ``` ### What the deployment must inject @@ -258,7 +328,7 @@ above and whatever delivers secrets: | Name | Kind | Consumed by | |---|---|---| -| `UNIFIED_API_KEY_AWX`, `UNIFIED_API_KEY_FORMS` | env | API authentication (`api_keys.yaml`) | +| `UNIFIED_API_KEY_AWX`, `UNIFIED_API_KEY_FORMS` | env | API authentication (the `api_keys` items) | | `D42_USERNAME`, `D42_PASSWORD` | env | `cred-d42` | | `NETBOX_TOKEN` | env | `cred-netbox` | | `FLEET_SSH_USERNAME` | env | `cred-fleet-ssh` | @@ -301,10 +371,20 @@ The config files become a ConfigMap (mounted at `CONFIG_DIR`), the env table becomes a Secret, the file secrets become Secret volume mounts, and the writable state gets a PVC. The complete manifests — Deployment with probes, non-root and resources, Service, PVC — live in -[`deploy/k8s/`](../deploy/k8s/), linted with kubeconform in CI: +[`deploy/k8s/`](../deploy/k8s/), linted with kubeconform in CI. + +A ConfigMap is **flat**: `--from-file` does not recurse, and a key cannot hold a +`/`. The configuration is item files in folders (`sources/`, `connectors/`...), +but the loader does not care where a file sits — the top-level key inside says +what it holds — so flatten the tree into the keys, as below, or give each key a +`path:` under `items:` in the volume to rebuild the folders. A directory that +changes often is better pushed with the [configuration API](config-api.md) onto +a PVC. ```bash -kubectl create configmap unified-api-config --from-file=config/ +# config/sources/src-a.yaml becomes the key sources-src-a.yaml +kubectl create configmap unified-api-config $(cd config && find . -name '*.yaml' \ + | sed 's|^\./||' | while read -r f; do printf -- '--from-file=%s=config/%s ' "${f//\//-}" "$f"; done) kubectl create secret generic unified-api-env --from-env-file=.env kubectl create secret generic unified-api-secrets \ --from-file=ssh-private-key=secrets/id_ed25519 \ @@ -365,13 +445,18 @@ What changes when the deployment target is a cluster rather than a single host. - **Probes:** liveness → `GET /healthz`; readiness → `GET /readyz` (503 until at least one configured source has synced, so pods join the Service only with data) -- **Config:** mount a ConfigMap at a path and point `CONFIG_DIR` at it +- **Config:** mount a ConfigMap at a path and point `CONFIG_DIR` at it. It is + flat (see [Kubernetes](#kubernetes) above) and its files are symlinks into a + hidden `..data` folder — the loader follows the links and skips the hidden + folder, so a mounted ConfigMap loads as-is. It is also read-only, which + matters for the [config API](config-api.md) and for an [upgrade from + 0.34](#upgrading-from-034) - **Secrets:** three ways in. Inject values as env vars matching each credential's `env_prefix`/`secret_keys` (e.g. from a Secret via `envFrom`, populated by External Secrets Operator); mount files and reference them via `secret_file` / `file_keys`; or resolve natively from HashiCorp Vault — give a credential a `vault_path` and configure `secrets.vault:` (token or - Kubernetes auth), see [configuration](configuration.md#credentialsyaml). + Kubernetes auth), see [configuration](configuration.md#credentials). Adoption is per credential, so the three coexist during a migration - **API key:** set `UNIFIED_API_KEY` from a Secret; without it the API is open - **Replicas:** the cache is per-process and not shared. Multiple replicas each @@ -393,7 +478,7 @@ in an app-of-apps. A working layout: apps/unified-api/manifests/ ├── namespace.yaml ├── serviceaccount.yaml -├── configmap.yaml # config/*.yaml files, mounted at CONFIG_DIR +├── configmap.yaml # config.yaml + the item files (flattened), mounted at CONFIG_DIR ├── # see below ├── deployment.yaml # start from deploy/k8s/deployment.yaml in this repo ├── service.yaml # start from deploy/k8s/service.yaml @@ -471,6 +556,33 @@ writes back, or the app never reports Synced. --- +## Upgrading from 0.34 + +0.35 replaced the eight fixed configuration files with item files and moved each +source's script onto a connector item (see [configuration → upgrading from +0.34](configuration.md#upgrading-from-034) for the full mapping). What happens to +a deployment depends on whether its configuration directory is writable: + +| Configuration delivered by | At the first 0.35 start | What to do | +|---|---|---| +| A writable directory (a PVC, a host bind mount, systemd) | Converted **in place**, once; the old files move to `.legacy-config-/` | Commit the new files wherever the directory comes from, or the next push brings the old layout back | +| A ConfigMap, or a `:ro` bind mount (as in the `docker run` above) | Converted into a **private temporary copy** and loaded from there, with a loud warning. **Every restart converts again**, and config API writes land in the copy | Convert the source: `unified-api --migrate-config ./config` in the configuration repository, commit, regenerate the ConfigMap | +| A configuration pipeline pushing over the [config API](config-api.md) | The directory on disk converts as above; a push of old-layout files is then **refused**, naming `--migrate-config` | Convert the repository before upgrading the instances it pushes to | + +Either way the pod starts — refusing its own volume would brick an upgrade — +unless the conversion itself would not load, in which case it refuses to start +**having changed nothing**, and the log says why. Before rolling the image, run +`--check-config` on the configuration repository with the new version: on a 0.34 +directory it validates the conversion without writing (see +[CLI](cli.md#validating-configuration---check-config)). + +Rolling back to 0.34 needs the old files back: 0.34 cannot read item files. On a +writable directory they are in the backup folder; on a ConfigMap, redeploy the +0.34 one. The API itself did not change shape, so no consumer needs to change +in either direction. + +--- + ## Scheduling and observability How often things run, what is logged, and the Prometheus metrics to alert on diff --git a/docs/endpoints.md b/docs/endpoints.md index a25ad0b..476a9e5 100644 --- a/docs/endpoints.md +++ b/docs/endpoints.md @@ -5,7 +5,7 @@ more cached datasets through a transformer and returns whatever the consumer needs — the builtin `ansible` transformer renders a merged Ansible inventory for AWX and AnsibleForms. A transformer is either a **builtin** (`output:`, run in-process) or an external **script** (`script_path:`). Field reference lives in -[configuration → endpoints.yaml](configuration.md#endpointsyaml). +[configuration → endpoints](configuration.md#endpoints). - [What an endpoint is](#what-an-endpoint-is) - [Builtin transformers](#builtin-transformers) @@ -25,12 +25,14 @@ never writes to the cache — it renders what is already there, at request time, every time. ```yaml -ep-awx-full: - name: "Full AWX inventory" - source_ids: ["src-d42", "src-fleet-facts", "src-inventory"] - output: ansible # a builtin — or script_path: for a script - config: # static transformer settings - filter: "os=OracleLinux" +# endpoints/ep-awx-full.yaml +endpoints: + - name: ep-awx-full + description: "Full AWX inventory" + sources: ["src-d42", "src-fleet-facts", "src-inventory"] + output: ansible # a builtin — or script_path: for a script + config: # static transformer settings + filter: "os=OracleLinux" ``` Endpoints read cached **sources**, never views — a view has no cache entry of @@ -115,7 +117,7 @@ and anything structured as compact JSON, quoted per RFC 4180. Renders are deterministic — identical inventory renders byte-for-byte identically, so responses diff cleanly across instances and across time. -The script-only knobs (`script_path`, `script_args`, `project_id`, +The script-only knobs (`script_path`, `script_args`, `project`, `timeout_seconds`) are config errors on a builtin endpoint, named at load — a builtin runs in-process, so none of them can mean anything. @@ -129,12 +131,13 @@ it keeps arrives with every variable, group and membership the other sources gave it. ```yaml -ep-awx-managed: - name: "Only what the CMDB manages" - source_ids: ["src-cmdb", "src-vmware", "src-facts"] - output: ansible - limit: - by_hosts_from_inventory: "src-cmdb" +endpoints: + - name: ep-awx-managed + description: "Only what the CMDB manages" + sources: ["src-cmdb", "src-vmware", "src-facts"] + output: ansible + limit: + by_hosts_from_inventory: "src-cmdb" ``` That endpoint merges three sources and returns the hosts `src-cmdb` lists — @@ -145,7 +148,7 @@ inventory: one source decides *who is in*, the others decide *what is known*. | Rule | Effect | |---|---| -| `by_hosts_from_inventory` | Keep only the hosts this source has — the same hosts `GET /sources/{id}/hosts` returns for it. It must be one of the endpoint's `source_ids` | +| `by_hosts_from_inventory` | Keep only the hosts this source has — the same hosts `GET /sources/{id}/hosts` returns for it. It must be one of the endpoint's `sources` | Three things worth knowing: @@ -167,7 +170,7 @@ Three things worth knowing: emptied is "nothing", and the group goes — leaving, like any dropped group, a `children` entry on its parent that no longer resolves (see above). -Config errors, all named at load: a limit naming a source outside `source_ids` +Config errors, all named at load: a limit naming a source outside `sources` (the intersection would be against data the endpoint never reads and never waits for), a `limit:` with no rule in it, and a misspelled rule name. @@ -190,7 +193,7 @@ becomes the HTTP response body verbatim. | stdout | The response body, as-is. Output starting with `{` or `[` is served as `application/json`, anything else as `text/plain` | The script decides the format entirely — inventory JSON, INI, CSV, plain -text. With `project_id` set, the script path resolves inside the checkout at +text. With `project` set, the script path resolves inside the checkout at every execution and the project's [virtualenv](projects.md#python-virtualenvs) leads its PATH. @@ -233,7 +236,7 @@ cannot read the underlying sources raw. The endpoint is the product: a consumer given `ep-awx-full` gets the merged inventory without also getting `GET /sources/{id}/dataset` on the members. -Restricted keys list endpoint ids under `endpoints:` in `api_keys.yaml`; +Restricted keys list endpoint ids under `endpoints:` in their `api_keys` item; `GET /api/v1/endpoints` filters to what the key may run and reports each endpoint's `sources_ready` / `sources_missing`. diff --git a/docs/enrichers.md b/docs/enrichers.md index 283ac63..2edd6ca 100644 --- a/docs/enrichers.md +++ b/docs/enrichers.md @@ -16,7 +16,7 @@ treatment; the script I/O contract also appears in [connectors](connectors.md#en ## Two modes -Both modes write into the cache entry of their `target_id`; they differ in +Both modes write into the cache entry of their `target`; they differ in where the new values come from and how much code you owe. **Script-based** — a process receives the target's current dataset on stdin @@ -24,28 +24,36 @@ and prints a *partial* dataset of changes. Use it when the enrichment needs logic or the outside world (a DNS lookup, an HTTP probe): ```yaml -enrich-reachability: - name: "Probe reachability" - target_id: "src-fleet-facts" - script_path: "enrichers/probe.py" - project_id: "prj-connectors" # optional: resolve the script (and its - # virtualenv) inside this checkout - script_args: ["--timeout", "5"] - sync_interval_seconds: 900 - timeout_seconds: 300 +# enrichers/enrich-reachability.yaml +enrichers: + - name: enrich-reachability + description: "Probe reachability" + target: "src-fleet-facts" + project: "prj-connectors" # optional: resolve the script (and its + # virtualenv) inside this checkout + script_path: "enrichers/probe.py" + script_args: ["--timeout", "5"] + sync_interval_seconds: 900 + timeout_seconds: 300 ``` +An enricher runs its own script rather than naming a [connector](connectors.md): +it gathers nothing, so it has no system to talk to — hence `script_path` here +where a connector has `path`. + **Declarative merge** — no script at all: copy the named `fields` from one cached source onto the hosts of another. Use it for the common "join two sources by hostname" case: ```yaml -enrich-storage: - name: "Attach storage volumes" - target_id: "src-d42" # who receives the fields - source_id: "src-purestorage" # who provides them - fields: ["volumes", "array"] - sync_interval_seconds: 900 +# enrichers/enrich-storage.yaml +enrichers: + - name: enrich-storage + description: "Attach storage volumes" + target: "src-d42" # who receives the fields + source: "src-purestorage" # who provides them + fields: ["volumes", "array"] + sync_interval_seconds: 900 ``` A declarative enricher writes **only** the fields it names — never the rest of @@ -116,7 +124,7 @@ vars reaching it — it does not remove a group the target owns. Only **variables** cross. A source cannot pull its own hosts into the target through a shared group name, so an enricher stays safe where adding the source -to the endpoint's `source_ids` would not be. +to the endpoint's `sources` would not be. --- @@ -144,8 +152,8 @@ not mention are untouched — concurrent writes to them are never lost. > so you can carry the rest through. See > [troubleshooting](troubleshooting.md#a-script-enrichers-keys-keep-disappearing). -With `project_id` set, the script path resolves inside that project's checkout -at every execution, and if the project has a [virtualenv](configuration.md#projectsyaml) +With `project` set, the script path resolves inside that project's checkout +at every execution, and if the project has a [virtualenv](configuration.md#projects) its `bin/` leads the script's PATH — pip-installed imports work. --- @@ -217,4 +225,4 @@ governs is the **target's** — there is no separate enricher grant to manage. | `POST /api/v1/enrichers/{id}/run` | Run now; `404` for an unknown id **or** an uncached target — the body names which | Configuration field reference lives in -[configuration → enrichers.yaml](configuration.md#enrichersyaml). +[configuration → enrichers](configuration.md#enrichers). diff --git a/docs/federation.md b/docs/federation.md index 473bb74..4cf1dc8 100644 --- a/docs/federation.md +++ b/docs/federation.md @@ -4,7 +4,7 @@ One instance per datacenter doing the local gathering, one central that federates them — so consumers only ever talk to the central, and no SSH key or firewall opening needs global reach. -This page is the topology. The `connector_type: remote` script contract it rests +This page is the topology. The built-in `remote` connector it rests on lives in [connectors](connectors.md), and [views](views.md) are how a consumer stops needing to know which instance a host sits behind. @@ -18,7 +18,7 @@ For hosts spread over multiple datacenters, don't SSH across the WAN from one central instance (firewall openings into every DC, one key with global reach, WAN latency on every handshake). Deploy **one instance per DC** doing the local work, and **one central** that federates them with -`connector_type: "remote"` — consumers only ever talk to the central: +the built-in `connector: remote` — consumers only ever talk to the central: ``` DATACENTER A DATACENTER B @@ -74,12 +74,13 @@ the worked example above). The only federation-specific piece is a **restricted API key** for the central: ```yaml -# edge: api_keys.yaml -key-central: - name: "Central aggregator" - env: "UNIFIED_API_KEY_CENTRAL" - # restricted (default role): the central can read THIS source and nothing else - sources: ["src-fleet"] +# edge: api_keys/key-central.yaml +api_keys: + - name: key-central + description: "Central aggregator" + env: "UNIFIED_API_KEY_CENTRAL" + # restricted (default role): the central can read THIS source and nothing else + sources: ["src-fleet"] ``` The deployment injects `UNIFIED_API_KEY_CENTRAL` on the edge (same secret @@ -92,45 +93,44 @@ mechanisms as everything else). Generate one distinct key per edge. What the central instance needs in order to treat each edge as a remote source it can read. ```yaml -# central: credentials.yaml — one token credential per DC -cred-edge-dc1: - name: "Edge datacenter A API key" - type: "token" - env_prefix: "EDGE_DC1" - secret_keys: - token: "TOKEN" # reads env EDGE_DC1_TOKEN +# central: credentials/cred-edge-dc1.yaml — one token credential per DC +credentials: + - name: cred-edge-dc1 + description: "Edge datacenter A API key" + type: "token" + env_prefix: "EDGE_DC1" + secret_keys: + token: "TOKEN" # reads env EDGE_DC1_TOKEN ``` ```yaml -# central: sources.yaml — one remote source per DC -src-dc1: - name: "Datacenter A" - connector_type: "remote" - project_id: "prj-unused" # required by schema; unused by remote - script_path: "src-fleet" # the source id ON THE EDGE - credential_ids: ["cred-edge-dc1"] - sync_interval_seconds: 120 # how often the central re-pulls the edge - ttl_seconds: 600 - config: - url: "https://unified-api-dc1.example.com" - # http_timeout_seconds: "30" # default 30 - # insecure_tls: "true" # only for self-signed edges; opt-in +# central: sources/src-dc1.yaml — one remote source per DC +sources: + - name: src-dc1 + description: "Datacenter A" + connector: "remote" # built in: no connector item to write + credentials: ["cred-edge-dc1"] + sync_interval_seconds: 120 # how often the central re-pulls the edge + ttl_seconds: 600 + config: + url: "https://unified-api-dc1.example.com" + source: "src-fleet" # the source id ON THE EDGE + # http_timeout_seconds: "30" # default 30 + # insecure_tls: "true" # only for self-signed edges; opt-in ``` -```yaml -# central: projects.yaml — the stub the schema requires -prj-unused: - name: "unused" - git_url: "https://example.invalid/unused.git" - sync_on_boot: false -``` +`config.url` and `config.source` are both required. Before 0.35 the edge's +source id rode in `script_path`, and a remote source had to name a stub project +it never used; the built-in connector needs neither (a 0.34 central is converted +that way on upgrade — see [configuration](configuration.md#upgrading-from-034)). ```yaml -# central: endpoints.yaml — one merged world view for consumers -ep-global: - name: "Global inventory" - source_ids: ["src-dc1"] # add one id per DC - script_path: "tests/adapters/out/output/ansible_inventory.py" +# central: endpoints/ep-global.yaml — one merged world view for consumers +endpoints: + - name: ep-global + description: "Global inventory" + sources: ["src-dc1"] # add one id per DC + script_path: "tests/adapters/out/output/ansible_inventory.py" ``` Secrets the central's deployment must inject: `EDGE_DC1_TOKEN` (the value of @@ -151,11 +151,14 @@ pattern an SSH source already gathers by. On the edge that means the claim is maintained exactly once, where the gathering is configured: ```yaml -# edge: sources.yaml — the source states what it owns -src-fleet: - # ... connector config ... - advertise_scope: - groups: ["dc1"] # optional when hosts_from_source already says it +# edge: sources/src-fleet.yaml — the source states what it owns +sources: + - name: src-fleet + connector: "ssh-facts" + # ... credentials, config, TTLs ... + ttl_seconds: 600 + advertise_scope: + groups: ["dc1"] # optional when hosts_from_source already says it ``` The central's remote syncs fetch the edge's `/scope` alongside the dataset, @@ -201,7 +204,7 @@ Failure modes, all loud: |---|---| | `answered 401` | The token credential isn't the edge's API key | | `answered 403` | The edge key exists but isn't allowed that source id | -| `answered 404` | Wrong remote source id, or the edge hasn't synced it yet | +| `answered 404` | Wrong `config.source` (the remote source id), or the edge hasn't synced it yet | | `request … failed` (network) | WAN/DNS/TLS problem — the central keeps serving its last good copy | | WARN `could not read remote ages` | Data arrived fine; only the age lookup failed (treated as fresh) | @@ -232,7 +235,7 @@ curl -s -H "X-API-Key: $KEY" https://central.example.com/api/v1/topology?depth=2 } ``` -- **Peers come from configuration**: every `connector_type: remote` source, +- **Peers come from configuration**: every `connector: remote` source, grouped by its `config.url`. No registry to keep in sync. - **Each peer is asked in turn**, with the token credential its sources already use, and nests its own answer under `instance`. `depth` (default 2, @@ -251,7 +254,42 @@ curl -s -H "X-API-Key: $KEY" https://central.example.com/api/v1/topology?depth=2 - A restricted key sees each peer's version, not the peer's own peers: those were asked with this instance's key, not the caller's. -The UI's Topology page draws this as a tree. +The UI's Topology page draws this as a tree. Each line is green where data gets through, red where +the peer is unreachable or its pulls fail. + +## The map on an edge: pushed down from the central + +An edge reads from nobody, so its own `GET /topology` is just itself. The central fills the gap: + +- **Every minute** an instance with remote sources maps the federation (depth 2) and sends it to + each peer that answered: `PUT /api/v1/topology/federation`, with the key its remote sources use. + When two sources read one edge with different keys, the push uses the one the map asked with: + the first source by id. +- **What an edge is sent is redacted** the way a restricted caller's map is. It shows every + peer's name and version, but not that peer's own map (asked with the central's keys). +- **Errors stay on the central.** Why a peer was not asked is generic, and a failing link says it + fails (`consecutive_failures`) without its error: those name the central's env vars and Vault + paths. +- **No edge configuration.** The central's key is restricted to the sources it reads, and that is + enough: any key may push, and the map is stored under the key's name (`key-central` → central). +- **Only a key sent with the request pushes.** A browser session cannot, even one signed in with + a key: the answer is a `403`. +- **The central's address comes from its `server.url`** (`config.yaml`), since a process only + knows the port it listens on. Unset, the edge shows the central without a URL. +- **Admins read it** at `GET /api/v1/topology/federation`. The edge's Topology page then draws the + central at the root and this instance among its peers, and the Overview counts the same map. +- **In memory, and it expires.** A restart empties it until the next push (within a minute); a + sender silent for ten minutes is dropped. Past three minutes the page says the map may be stale. +- **Older edges are skipped quietly**: a 404 or 405 to the push is not a failure. Other failures + are logged once when they start, and once when they stop. + +```mermaid +flowchart LR + C[central] -- "GET /topology (every minute)" --> E1[edge dc1] + C -- "GET /topology" --> E2[edge dc2] + C -- "PUT /topology/federation: the map, redacted" --> E1 + C -- "PUT /topology/federation" --> E2 +``` ## Operational notes @@ -262,7 +300,7 @@ How the arrangement behaves once it is running, and what it costs to change its metric flags the broken link — alert on that. - **An unchanged edge costs almost nothing to poll**: full pulls revalidate with the edge's `ETag` and a `304` skips the transfer — see - [connectors → remote](connectors.md#remote-sources--federation-connector_type-remote). + [connectors → remote](connectors.md#remote-sources--federation-connector-remote). `unified_api_remote_not_modified_total` counts the skips, so a low ratio of skips to syncs on a source that rarely changes is worth a look. - **Adding a DC** = deploy an edge (same manifests, different config), give diff --git a/docs/ldap-login.md b/docs/ldap-login.md index 939a420..e162cdd 100644 --- a/docs/ldap-login.md +++ b/docs/ldap-login.md @@ -3,7 +3,7 @@ The browsable UI's login form can take a directory account (LDAP or Active Directory) as well as an API key. People log in with the username and password they already have; what they may touch is decided by **access rules in this -service's configuration**, in the same vocabulary `api_keys.yaml` uses. +service's configuration**, in the same vocabulary the `api_keys` items use. The directory says who someone is and which groups they belong to. It never decides access by itself: a new group in AD grants nothing until a rule names @@ -21,7 +21,7 @@ server: home: "/" # where a sign-in lands; see "Sessions" ldap: url: "ldaps://ldap.example.com:636" # or ldap://...:389 with starttls: true - bind_credential: cred-ldap # credentials.yaml: username = bind DN, password + bind_credential: cred-ldap # a credentials item: username = bind DN, password search_base: "OU=Users,DC=example,DC=com" # user_filter: "(sAMAccountName={username})" # the default (Active Directory) # group_attribute: memberOf # the default @@ -39,14 +39,16 @@ server: endpoints: [ep-ansible-vmware] ``` -The service account comes from `credentials.yaml` like any connector -credential, so it can live in env vars, a file or Vault: +The service account is a `credentials` item like any source's credential, so it +can live in env vars, a file or Vault: ```yaml -cred-ldap: - name: "LDAP service account" - type: username_password - vault_path: unified-api/ldap # or env_prefix / secret_file +# credentials/cred-ldap.yaml +credentials: + - name: cred-ldap + description: "LDAP service account" + type: username_password + vault_path: unified-api/ldap # or env_prefix / secret_file ``` Its `username` is the **bind DN** (`CN=svc-unified,OU=Service,DC=example,DC=com`) @@ -72,7 +74,7 @@ attribute. - `ldap://` without `starttls: true`, since every password would cross the network in clear text; -- a `bind_credential` that is not in `credentials.yaml`; +- a `bind_credential` that names no `credentials` item; - a `user_filter` without `{username}`; - an empty `access` list, or a rule naming no groups and no users; - a rule naming a source or endpoint that does not exist; diff --git a/docs/observability.md b/docs/observability.md index 696eade..b8b292b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -113,7 +113,7 @@ probes — scrapers don't carry the API key): | `unified_api_http_requests_total` | `method`, `path`, `status` | Every HTTP request, labeled by the **matched route pattern** (`/api/v1/sources/{id}/dataset`), never the raw URL — one series per route, not per host. Unrouted requests share `path="unmatched"` | | `unified_api_http_request_duration_seconds` | `method`, `path` | Request latency histogram — handler time, measured inside the gzip layer | | `unified_api_sync_total` | `source`, `result` | Sync runs: `success`, `error`, or `coalesced` (answered by a concurrent full sync's result — no gather ran) | -| `unified_api_remote_not_modified_total` | `url`, `source` | Federation pulls answered `304 Not Modified` — the transfer and re-parse were skipped, the sync still succeeded (see [connectors → remote](connectors.md#remote-sources--federation-connector_type-remote)) | +| `unified_api_remote_not_modified_total` | `url`, `source` | Federation pulls answered `304 Not Modified` — the transfer and re-parse were skipped, the sync still succeeded (see [connectors → remote](connectors.md#remote-sources--federation-connector-remote)) | | `unified_api_sync_duration_seconds` | `source` | Sync duration histogram | | `unified_api_refresh_total` | `source`, `result` | On-demand refreshes triggered by reads (`?refresh=true`): `fresh` (the cache answered, no gather ran), `coalesced`, `refreshed`, `failed` or `timeout` — how much gathering load comes from consumers rather than the scheduler (see [on-demand refresh](on-demand-refresh.md)) | | `unified_api_enrich_total` | `source`, `result` | Enricher runs | diff --git a/docs/on-demand-refresh.md b/docs/on-demand-refresh.md index 83ce021..51bde25 100644 --- a/docs/on-demand-refresh.md +++ b/docs/on-demand-refresh.md @@ -99,7 +99,7 @@ One request, followed from the consumer through the central to the origin and ba ┌────────────────────────────────────────────────────────┐ │ unified-api CENTRAL (datacenter A, k8s) │ │ │ - │ src-edge-dc2: connector_type: remote │ + │ src-edge-dc2: connector: remote │ │ allow_on_demand_refresh: true │ │ ttl_seconds: 300 │ │ │ @@ -119,7 +119,7 @@ One request, followed from the consumer through the central to the origin and ba ┌────────────────────────────────────────────────────────┐ │ unified-api EDGE (datacenter B, podman) │ │ │ - │ src-ssh-facts: connector_type: ssh │ + │ src-ssh-facts: connector: ssh-facts (type: ssh) │ │ hosts_from_source: src-inventory │ │ │ │ host scope honoured: SSH to node100 ONLY │ @@ -170,60 +170,77 @@ central's key for that source — which a **restricted** key does, since `POST Complete and copy-pasteable. Two instances, one datacenter each. -### Edge — `sources.yaml` (datacenter B, podman) +### Edge — connectors and sources (datacenter B, podman) Nothing new here. The edge is unchanged by this feature; it only has to honour a host-scoped sync, which it does as of 0.8.0. ```yaml -# The inventory that says WHAT exists in this DC -src-inventory: - name: "Device42 inventory (datacenter B)" - project_id: "prj-connectors" - script_path: "d42_inventory.py" - script_args: ["--list"] - output_format: "ansible" - connector_type: "script" - credential_ids: ["cred-d42"] - sync_interval_seconds: 3600 - ttl_seconds: 7200 - -# The facts: WHAT each host is doing. Host list comes from src-inventory. -src-ssh-facts: - name: "Ansible local facts (datacenter B)" - project_id: "prj-connectors" - script_path: "unused-in-facts-mode" - connector_type: "ssh" - credential_ids: ["cred-ssh-dc2"] - hosts_from_source: - source: "src-inventory" - match_pattern: - groups: ["dc2"] - connect_via: "ansible_host_then_hostname" - sync_interval_seconds: 1800 - # The edge's own TTL. Independent of the central's: this one describes when - # the edge's scheduler considers its data old, the central's decides when a - # consumer's read is worth a gather. - ttl_seconds: 3600 - config: - gather_mode: "facts" - fact_path: "/etc/ansible/facts.d" - concurrency: "40" - ssh_connect_timeout_seconds: "20" -``` - -### Edge — `api_keys.yaml` +# connectors/d42.yaml — an Ansible dynamic inventory script +connectors: + - name: d42 + description: "Device42 dynamic inventory" + project: "prj-connectors" + path: "d42_inventory.py" + script_args: ["--list"] + output_format: "ansible" + +# connectors/ssh-facts.yaml — native SSH, Ansible local facts +connectors: + - name: ssh-facts + description: "Ansible local facts over SSH" + type: "ssh" + path: "gather_facts" # a label: facts mode runs no command of ours +``` + +```yaml +# sources/src-inventory.yaml — the inventory that says WHAT exists in this DC +sources: + - name: src-inventory + description: "Device42 inventory (datacenter B)" + connector: "d42" + credentials: ["cred-d42"] + sync_interval_seconds: 3600 + ttl_seconds: 7200 + +# sources/src-ssh-facts.yaml — the facts: WHAT each host is doing. Host list +# comes from src-inventory. +sources: + - name: src-ssh-facts + description: "Ansible local facts (datacenter B)" + connector: "ssh-facts" + credentials: ["cred-ssh-dc2"] + hosts_from_source: + source: "src-inventory" + match_pattern: + groups: ["dc2"] + connect_via: "ansible_host_then_hostname" + sync_interval_seconds: 1800 + # The edge's own TTL. Independent of the central's: this one describes when + # the edge's scheduler considers its data old, the central's decides when a + # consumer's read is worth a gather. + ttl_seconds: 3600 + config: + gather_mode: "facts" + fact_path: "/etc/ansible/facts.d" + concurrency: "40" + ssh_connect_timeout_seconds: "20" +``` + +### Edge — `api_keys` The key the central uses. Restricted to the one source it federates: if the central is compromised it cannot read the rest of datacenter B's inventory. ```yaml -key-central: - name: "central (datacenter A)" - env: "UNIFIED_API_KEY_CENTRAL" - role: "restricted" - sources: ["src-ssh-facts"] - endpoints: [] +# api_keys/key-central.yaml +api_keys: + - name: key-central + description: "central (datacenter A)" + env: "UNIFIED_API_KEY_CENTRAL" + role: "restricted" + sources: ["src-ssh-facts"] + endpoints: [] ``` ### Central — `config.yaml` (datacenter A, k8s) @@ -246,49 +263,53 @@ cache: interval_seconds: 300 ``` -### Central — `sources.yaml` +### Central — `sources` + +The built-in `remote` connector: no connector item, and no project. ```yaml -src-edge-dc2: - name: "Datacenter B facts (federated)" - project_id: "prj-connectors" - # script_path = the source id ON THE EDGE - script_path: "src-ssh-facts" - connector_type: "remote" - credential_ids: ["cred-edge-dc2"] - # This is the gate. A read asking for a host refreshes it only if the host is - # older than this. Read the "Choosing the TTL" note before setting it. - ttl_seconds: 300 - # Off by default; this is what lets a GET trigger a gather. - allow_on_demand_refresh: true - # The scheduler keeps working as before: the central pulls the edge's cache - # on its own interval and NEVER causes SSH by doing so. - sync_interval_seconds: 600 - ttl_overrides: - hosts: - # A box whose state changes fast enough to be worth gathering more often - node100: 60 - groups: - # ...or a whole group of them - oracle_db: 120 - config: - url: "https://unified-api-dc2.example.com" - # Bounds BOTH the reads and the refresh POST, so it must exceed the time the - # edge needs to gather one host. Raise it on slow links or slow hosts. - http_timeout_seconds: "30" -``` - -### Central — `credentials.yaml` +# sources/src-edge-dc2.yaml +sources: + - name: src-edge-dc2 + description: "Datacenter B facts (federated)" + connector: "remote" + credentials: ["cred-edge-dc2"] + # This is the gate. A read asking for a host refreshes it only if the host is + # older than this. Read the "Choosing the TTL" note before setting it. + ttl_seconds: 300 + # Off by default; this is what lets a GET trigger a gather. + allow_on_demand_refresh: true + # The scheduler keeps working as before: the central pulls the edge's cache + # on its own interval and NEVER causes SSH by doing so. + sync_interval_seconds: 600 + ttl_overrides: + hosts: + # A box whose state changes fast enough to be worth gathering more often + node100: 60 + groups: + # ...or a whole group of them + oracle_db: 120 + config: + url: "https://unified-api-dc2.example.com" + source: "src-ssh-facts" # the source id ON THE EDGE + # Bounds BOTH the reads and the refresh POST, so it must exceed the time the + # edge needs to gather one host. Raise it on slow links or slow hosts. + http_timeout_seconds: "30" +``` + +### Central — `credentials` ```yaml -# The edge's API key, read from UNIFIED_API_EDGE_DC2_KEY. The remote connector -# looks for a credential named `token`, which is what secret_keys maps here. -cred-edge-dc2: - name: "Datacenter B edge API key" - type: "token" - env_prefix: "UNIFIED_API_EDGE_DC2" - secret_keys: - token: "KEY" +# credentials/cred-edge-dc2.yaml — the edge's API key, read from +# UNIFIED_API_EDGE_DC2_KEY. The remote connector looks for a credential named +# `token`, which is what secret_keys maps here. +credentials: + - name: cred-edge-dc2 + description: "Datacenter B edge API key" + type: "token" + env_prefix: "UNIFIED_API_EDGE_DC2" + secret_keys: + token: "KEY" ``` ### The secret contract @@ -472,9 +493,9 @@ Every way a refresh can fall short, and what the caller is told in each case. | Edge unreachable (WAN down) | `200` + cached data, `refreshed: false` | Same reason. Reads keep working off the last snapshot | | `refresh=true` with no `?host=` | `400` + explanation | Naming the hosts is what bounds the cost | | Source without `allow_on_demand_refresh` | `403` naming the setting | The capability was never granted | -| Source not in `sources.yaml` | `404` | Configuration error, not a cache miss | +| Source not configured | `404` | Configuration error, not a cache miss | | Central's key rejected by the edge | `200` + cached data, `refreshed: false`, error contains `401` | The key rotated on one side only | -| Central's key not scoped to the edge source | same, error contains `403` | `api_keys.yaml` on the edge | +| Central's key not scoped to the edge source | same, error contains `403` | the `api_keys` item on the edge | | Hostname differs between central and edge | `200`, `refreshed: false`, error says the scope matched no hosts | The two sides must key hosts identically | | `refresh_depth` runs out mid-chain | `200`, data served, `WARN` in the log | Chain longer than the budget, or a cycle | | Source has no cache entry at all | Refresh runs (nothing cached is maximally stale), then `404` if it still has nothing | A refresh cannot invent a source | diff --git a/docs/projects.md b/docs/projects.md index a535073..0c5bad5 100644 --- a/docs/projects.md +++ b/docs/projects.md @@ -4,7 +4,7 @@ Git repositories that carry the code the app executes: connector scripts, enrichers, output transformers, static inventories. A **project** is a checkout the app clones, keeps up to date, and resolves script paths into — so rolling a new script version is a `git push`, not an image rebuild. Field -reference lives in [configuration → projects.yaml](configuration.md#projectsyaml). +reference lives in [configuration → projects](configuration.md#projects). - [The lifecycle](#the-lifecycle) - [Boot never blocks on git](#boot-never-blocks-on-git) @@ -18,7 +18,7 @@ reference lives in [configuration → projects.yaml](configuration.md#projectsya ## The lifecycle -One directory per project id under `projects.dir` (default `./projects`), +One directory per project name under `projects.dir` (default `./projects`), created by a shallow clone and updated by fetch + hard reset — local drift in a checkout is deliberately discarded, because the repository is the truth. @@ -57,17 +57,26 @@ the first source has synced, exactly as without projects. ## How script paths resolve -A *relative* `script_path` whose file exists inside the checkout runs from -there (`//`); otherwise the configured -path is kept as-is, so image-baked and mounted scripts keep working. +Three kinds of item point into a checkout, and all say it the same way: + +| Item | Names the project | Names the file | +|---|---|---| +| A script [connector](connectors.md#connectors-and-sources) | `project` | `path` | +| A `static_inventory` source | `project` | `path` | +| An enricher or endpoint (optional) | `project` | `script_path` | + +A *relative* path whose file exists inside the checkout runs from there +(`//`); otherwise the configured path is kept +as-is, so image-baked and mounted scripts keep working. Resolution happens at **every execution**, not once at boot: a script that first appears after startup — a slow clone, a pipeline's first push to a new project — is used on the very next run, no restart. A checkout that exists but lacks the named file logs a warning and keeps the configured path. SSH -sources are never resolved; their `script_path` is a remote command. Sources -always declare `project_id`; enrichers and endpoints may add an optional one -to resolve their scripts the same way. +connectors are never resolved — their `path` is a remote command — which is why +they refuse a `project`. Remote sources run nothing locally and name no project +at all. `GET /api/v1/projects` lists under `used_by` the connectors, sources, +enrichers and endpoints that run from each checkout. --- @@ -111,13 +120,13 @@ curl -s -H "x-api-key: $ADMIN_KEY" -X POST $BASE/api/v1/projects/prj-connectors/ ## Private repositories -A `credential_id` on the project authenticates the clone and every pull — +A `credential` on the project authenticates the clone and every pull — without ever putting the secret on a command line (argv is world-readable in `/proc` while git runs). A `token` (or username/password) credential becomes a Basic-auth header passed through `GIT_CONFIG_*` environment variables; an `ssh_key` credential sets -`GIT_SSH_COMMAND` pointing at the key file. See [secrets](configuration.md#credentialsyaml) +`GIT_SSH_COMMAND` pointing at the key file. See [secrets](configuration.md#credentials) for how credentials resolve. --- diff --git a/docs/secrets.md b/docs/secrets.md index a5e8f04..f40ffbf 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -1,10 +1,10 @@ # Secrets How credentials resolve, rotate, and reach the scripts. The one principle -everything follows: **configuration never holds a secret** — `credentials.yaml` +everything follows: **configuration never holds a secret** — a `credentials` item describes *where* to read each value, and the infrastructure (env vars, mounted files, Vault) delivers it. Field reference lives in -[configuration → credentials.yaml](configuration.md#credentialsyaml). +[configuration → credentials](configuration.md#credentials). - [The three backends](#the-three-backends) - [Resolution order](#resolution-order) @@ -119,7 +119,7 @@ are not cached — the next attempt re-asks the backend. ## API keys are separate -The keys consumers present to *this* API live in `api_keys.yaml`, follow the +The keys consumers present to *this* API are `api_keys` items, follow the same never-in-YAML principle (each definition names the env var holding its secret), and are documented with the [API](api.md#authentication). diff --git a/docs/testing.md b/docs/testing.md index 7f4cdb1..1655b72 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -61,6 +61,8 @@ The integration files: | `tests/remote_connector_test.rs` | Federation: the remote connector against a stub upstream | | `tests/static_inventory_test.rs` | Ansible YAML inventories read from disk | | `tests/git_test.rs` | Project checkouts through the git CLI adapter | +| `tests/connectors_test.rs` | `GET /api/v1/connectors`: what each connector runs, the sources using it and their health, the built-ins, admin-only | +| `tests/migration_test.rs` | Upgrading a 0.34 directory in place: startup converts the fixture under `tests/fixtures/legacy-0.34/`, serves the same ids and keeps the old files in the backup; a conversion that would not load changes nothing and refuses to start; `--migrate-config` and `--check-config` on it (the read-only fallback is a unit test in `src/config/migrate.rs`) | | `tests/config_api_test.rs` | The configuration API: what a push reads, validates, writes and reloads — and the four things it refuses (a change that would not load, a stale `If-Match`, a reload that would remove authentication, a key whose env var is missing) | Integration tests build a real `AppState` via `AppBuilder`, which defaults to the diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index cbbe9ec..d15a4b7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -22,6 +22,7 @@ look like a dataset slowly getting older. Only the second pair tells them apart. - [A sync times out](#a-sync-times-out) - [Hosts vanish from a group after a sync](#hosts-vanish-from-a-group-after-a-sync) - [Everything answers 401 or 403](#everything-answers-401-or-403) +- [The pod will not start after an upgrade to 0.35](#the-pod-will-not-start-after-an-upgrade-to-035) - [A configuration push is refused or does not take effect](#a-configuration-push-is-refused-or-does-not-take-effect) - [Useful one-liners](#useful-one-liners) @@ -164,7 +165,7 @@ than patching it key by key. Whatever the script omits is dropped — including keys another enricher owns. The dataset arrives on its stdin precisely so it can carry the rest through. -A *declarative* enricher (`source_id` + `fields`) has no such constraint: it +A *declarative* enricher (`source` + `fields`) has no such constraint: it writes only the fields it owns. See [connectors](connectors.md). --- @@ -222,6 +223,22 @@ so loudly, and every caller is treated as admin. --- +## The pod will not start after an upgrade to 0.35 + +0.35 converts a 0.34 configuration directory at startup (see +[configuration → upgrading](configuration.md#upgrading-from-034)). It refuses to +start only when the converted configuration would not load — and then it has +changed **nothing**: the old files are where they were. + +| Log line | Cause and fix | +|---|---| +| `Refusing to start: the configuration is in the 0.34 layout and could not be migrated (nothing was changed): ...` | The rest of the line is the reason: usually a 0.34 configuration that was already invalid, or a target file that already exists (`... already exists — the migration will not overwrite it`). Fix it, or run `unified-api --migrate-config` on a copy to see the plan | +| `... mixes id-keyed items (the 0.34 layout) with lists (0.35) — convert it by hand` | One legacy-named file was half edited into the new shape. Pick one layout for it | +| `Configuration directory ... has been migrated` (WARN) | Not a failure: the conversion happened. Commit the new files wherever the directory comes from | +| `... read-only: it was converted into a temporary copy` (WARN) | Not a failure either, but it repeats on every restart. Convert the ConfigMap's source with `--migrate-config` | + +--- + ## A configuration push is refused or does not take effect Every refusal from the [configuration API](config-api.md) is deliberate and @@ -232,10 +249,13 @@ transactional — a rejected push touched nothing. |---|---| | `403` naming `config_api.enabled` | The API is off (the default). Set `config_api.enabled: true` in the mounted `config.yaml` and restart — it cannot be enabled over the API itself, on purpose | | `400` with an `errors` list | The staged directory did not validate — the same list `--check-config` prints, every problem at once. Nothing was written; fix and re-push | +| `400`: `unknown key '' ... this is the 0.34 layout` | The pipeline pushed files still in the 0.34 layout (items keyed by id). The instance converted its own directory at startup; the repository was not. Run `unified-api --migrate-config` on the repository, commit, push again — see [configuration → upgrading](configuration.md#upgrading-from-034) | +| `404` naming the path rule | `{path}` is not one the loader would read: it must be relative, end in `.yaml`/`.yml`, and use only letters, digits, `-`, `_` and `.` in segments that do not start with a dot | | `412` | Your `If-Match` no longer matches: someone else wrote the file (or the directory) first. `GET` it again for a fresh `ETag`, merge, retry | | `409` about API keys | The change would leave the API with **no keys at all** (silent auth removal), or names a key env var that is not set on the instance. Both are refused before anything commits | | `413` | The push exceeds `server.max_body_bytes` (default 2 MiB) — a whole-directory `PUT` is one body. Raise the key; it applies on a reload | | `200`, but nothing changed | The write landed on disk and was never applied: no `?reload=true` on the write, and nobody called `POST /config/reload`. `GET /api/v1/config` shows `reload_pending: true` for exactly this state | +| A push lands, and is gone after a restart | The directory is a temporary copy: a read-only `CONFIG_DIR` (a ConfigMap) still in the 0.34 layout is converted into one at every start, and that copy is what the API writes to. The startup log says so at WARN. Convert the source of the files and redeploy | | `restart_required` will not clear | The change touches a restart-only key (`server.port`, `cache.persistence`, …). It keeps being reported — by `GET /api/v1/config` and the `unified_api_config_restart_required` gauge — until a restart adopts it. That persistence is the design: restart the instance | The audit log records every write and reload (`action: config_write`, diff --git a/docs/views.md b/docs/views.md index 54aad56..cd4ec1b 100644 --- a/docs/views.md +++ b/docs/views.md @@ -9,7 +9,7 @@ Consumers get one address for "the facts" and stop needing to know which datacenter — and therefore which source — a host lives in. - [The problem it solves](#the-problem-it-solves) -- [Configuration — `views.yaml`](#configuration--viewsyaml) +- [Configuration — `views` items](#configuration--views-items) - [Where it answers](#where-it-answers) - [Ownership is declared, not inferred](#ownership-is-declared-not-inferred) - [Advertised ownership](#advertised-ownership) @@ -22,7 +22,7 @@ datacenter — and therefore which source — a host lives in. ## The problem it solves -[Federation](on-demand-refresh.md) solved half of this. `connector_type: remote` +[Federation](on-demand-refresh.md) solved half of this. `connector: remote` means a central needs no credentials and no SSH path into a datacenter: the edge that owns the hosts does the gathering, and the central holds its dataset. @@ -64,26 +64,32 @@ Right shape, wrong mechanism. A view filters in Rust and keeps refresh. --- -## Configuration — `views.yaml` +## Configuration — `views` items -A view is declared like any other entity, naming the sources it merges. +A view is declared like any other item, under `views:` in any file of the +configuration directory, naming the sources it merges. ```yaml -vw-facts-all: - name: "Facts, both datacenters" - members: - - source: "src-ssh-dc1" # where the facts come from - owns: - source: "src-d42" # inventory the pattern resolves against - groups: ["datacenter_dc1"] - - source: "src-edge-dc2" # a remote (federated) member works too - owns: - source: "src-d42" - groups: ["datacenter_dc2"] - hosts: ["appliance01.dc2.example"] # claimed literally as well - # ttl_seconds: 30 # optional; absent = inherit the owner's +# views/vw-facts-all.yaml +views: + - name: vw-facts-all # the id consumers read, on the source routes + description: "Facts, both datacenters" + members: + - source: "src-ssh-dc1" # where the facts come from + owns: + source: "src-d42" # inventory the pattern resolves against + groups: ["datacenter_dc1"] + - source: "src-edge-dc2" # a remote (federated) member works too + owns: + source: "src-d42" + groups: ["datacenter_dc2"] + hosts: ["appliance01.dc2.example"] # claimed literally as well + # ttl_seconds: 30 # optional; absent = inherit the owner's ``` +A view's `name` shares the source id space: a view and a source with the same +name fail startup, since both would answer on `/api/v1/sources/{id}`. + | Field | Meaning | |---|---| | `members` | Ordered. The **first** member that claims a host wins it | @@ -97,10 +103,10 @@ An `owns` with neither `groups` nor `hosts` is a **catch-all**: everything its ownership source knows. Useful as a last member ("whatever the others did not claim"), dangerous as a first one — hence the ordering rule being explicit. -Unknown keys inside a view are a hard startup error, unlike the rest of the -config. Ownership is the routing table: `grups:` instead of `groups:` would -otherwise deserialize into an empty pattern, and an empty pattern claims -everything. +Unknown keys inside a view are a hard startup error, as they are everywhere in +the configuration — and here it matters most. Ownership is the routing table: +`grups:` instead of `groups:` would otherwise deserialize into an empty pattern, +and an empty pattern claims everything. --- @@ -135,15 +141,16 @@ facts from there. ### Auth -A view is granted exactly like a source, by its id under `sources:` in -`api_keys.yaml`. A key granted the view needs **no** access to the members: the +A view is granted exactly like a source, by its id under `sources:` in an +`api_keys` item. A key granted the view needs **no** access to the members: the view is the contract, the members are internal topology. ```yaml -key-forms: - name: "AnsibleForms" - env: "UNIFIED_API_KEY_FORMS" - sources: ["vw-facts-all"] # not the members +api_keys: + - name: key-forms + description: "AnsibleForms" + env: "UNIFIED_API_KEY_FORMS" + sources: ["vw-facts-all"] # not the members ``` --- diff --git a/src/adapters/in/http/auth.rs b/src/adapters/in/http/auth.rs index db6a042..b894cb5 100644 --- a/src/adapters/in/http/auth.rs +++ b/src/adapters/in/http/auth.rs @@ -11,6 +11,10 @@ use subtle::ConstantTimeEq; // (main.rs does the resolution at the boundary; nothing here touches env vars). #[derive(Clone)] pub struct ResolvedApiKey { + // The key's identity: its `name` in the api_keys item, what a peer is + // recorded under when it pushes the federation map. + pub id: String, + // Its label (`description`, falling back to the name): logs, Users page. pub name: String, pub secret: String, pub permissions: Permissions, @@ -127,6 +131,7 @@ pub fn resolve_api_keys(cfg: &crate::config::AppConfig) -> Result Result, + // The key's identity (its api_keys name), when a key was presented + // with the request; None open, or through a browser session. + pub key_id: Option, pub permissions: Permissions, pub method: AuthMethod, } @@ -267,6 +276,7 @@ pub async fn require_api_key( if keys.is_empty() { request.extensions_mut().insert(AuthContext { key_name: None, + key_id: None, permissions: Permissions::Admin, method: AuthMethod::Open, }); @@ -345,6 +355,7 @@ pub async fn require_api_key( request.extensions_mut().insert(identity.clone()); request.extensions_mut().insert(AuthContext { key_name: Some(auth.key_name), + key_id: None, permissions: auth.permissions, method: match auth.kind { crate::adapters::r#in::http::session::SessionKind::ApiKey => AuthMethod::ApiKey, @@ -373,6 +384,7 @@ pub async fn require_api_key( tracing::Span::current().record("key_name", key.name.as_str()); request.extensions_mut().insert(AuthContext { key_name: Some(key.name.clone()), + key_id: Some(key.id.clone()), permissions: key.permissions.clone(), method: AuthMethod::ApiKey, }); @@ -400,7 +412,7 @@ mod tests { .unwrap(); std::fs::write( dir.path().join("credentials.yaml"), - "cred-ldap:\n name: \"LDAP\"\n type: username_password\n env_prefix: \"LDAP\"\n", + "credentials:\n - name: cred-ldap\n description: \"LDAP\"\n type: username_password\n env_prefix: \"LDAP\"\n", ) .unwrap(); let cfg = crate::config::load_config(dir.path().to_str().unwrap()).unwrap(); @@ -409,6 +421,7 @@ mod tests { assert!(err.contains("no authentication"), "{err}"); let key = ResolvedApiKey { + id: "admin".to_string(), name: "admin".to_string(), secret: "s".to_string(), permissions: Permissions::Admin, @@ -431,6 +444,7 @@ mod tests { fn admin_key(name: &str, secret: &str) -> ResolvedApiKey { ResolvedApiKey { + id: name.to_string(), name: name.to_string(), secret: secret.to_string(), permissions: Permissions::Admin, diff --git a/src/adapters/in/http/config.rs b/src/adapters/in/http/config.rs index 984df6a..9ba8edf 100644 --- a/src/adapters/in/http/config.rs +++ b/src/adapters/in/http/config.rs @@ -11,7 +11,7 @@ use utoipa::ToSchema; use crate::AppState; use crate::adapters::out::config::fs::directory_etag; use crate::application::config as reload_use_case; -use crate::config::{AppConfig, CONFIG_FILES, ConfigErrors, REQUIRED_CONFIG_FILE, is_config_file}; +use crate::config::{AppConfig, ConfigErrors, REQUIRED_CONFIG_FILE, is_config_path}; use crate::ports::config_store::{ConfigChange, ConfigFileStat, ConfigStorePort}; use super::audit; @@ -49,6 +49,7 @@ pub struct ConfigFileInfo { #[derive(Serialize, ToSchema)] pub struct ConfigSummary { pub sources: usize, + pub connectors: usize, pub views: usize, pub credentials: usize, pub enrichers: usize, @@ -115,6 +116,7 @@ pub struct ReloadInfo { #[serde(skip_serializing_if = "Vec::is_empty")] pub restart_required: Vec, pub sources: DeltaInfo, + pub connectors: DeltaInfo, pub views: DeltaInfo, pub enrichers: DeltaInfo, pub endpoints: DeltaInfo, @@ -192,7 +194,8 @@ pub struct ReloadQuery { // of its TTL — the console outliving the revocation, which is the one thing a // revocation is for. fn swap_keys(state: &AppState, keys: &ApiKeys, resolved: Vec) -> usize { - let live: Vec = resolved.iter().map(|key| key.name.clone()).collect(); + // By identity: a key's label (its description) need not be unique. + let live: Vec = resolved.iter().map(|key| key.id.clone()).collect(); let count = resolved.len(); keys.0.replace(resolved); state.sessions.retain_keys(&live); @@ -219,14 +222,18 @@ fn admin(auth: &AuthContext) -> Result<(), ApiError> { } } +// A path the loader would read: relative, at any depth, safe segments, a +// YAML extension. Anything else could never be part of the configuration, so +// it is refused before it is read or written. fn known_file(name: &str) -> Result<(), ApiError> { - if is_config_file(name) { + if is_config_path(name) { Ok(()) } else { Err(ApiError::not_found(format!( - "'{}' is not a configuration file — the loader reads exactly: {}", - name, - CONFIG_FILES.join(", ") + "'{}' is not a configuration path — a relative path ending in .yaml or \ + .yml, whose segments use only letters, digits, '-', '_' and '.' and \ + do not start with a dot", + name ))) } } @@ -247,6 +254,7 @@ fn file_info(stat: &ConfigFileStat) -> ConfigFileInfo { fn summarize(cfg: &AppConfig) -> ConfigSummary { ConfigSummary { sources: cfg.sources.len(), + connectors: cfg.connectors.len(), views: cfg.views.len(), credentials: cfg.credentials.len(), enrichers: cfg.enrichers.len(), @@ -262,6 +270,7 @@ fn reload_info(report: &reload_use_case::ReloadReport, api_keys: usize) -> Reloa applied: report.applied.clone(), restart_required: report.restart_required.clone(), sources: (&report.sources).into(), + connectors: (&report.connectors).into(), views: (&report.views).into(), enrichers: (&report.enrichers).into(), endpoints: (&report.endpoints).into(), @@ -380,12 +389,13 @@ pub async fn get_config( let store = store(&state)?; let stats = store.stat_all().map_err(ApiError::internal)?; - let present: Vec<&str> = stats.iter().map(|s| s.name.as_str()).collect(); - let missing = CONFIG_FILES - .iter() - .filter(|name| !present.contains(name)) - .map(|name| name.to_string()) - .collect(); + // Only one file is ever required; every other is an item file that is + // there or not. + let missing = if stats.iter().any(|s| s.name == REQUIRED_CONFIG_FILE) { + Vec::new() + } else { + vec![REQUIRED_CONFIG_FILE.to_string()] + }; // What is on disk may differ from what is running (somebody wrote without // reloading) and may not even load (somebody edited the file by hand). @@ -421,7 +431,7 @@ pub async fn get_config( get, path = "/api/v1/config/{file}", tag = "Configuration", - params(("file" = String, Path, description = "File name, e.g. sources.yaml")), + params(("file" = String, Path, description = "Path inside the configuration directory, e.g. sources/acme/src-d42-acme.yaml (slashes allowed)")), responses( (status = 200, description = "The file, verbatim. The ETag is the sha256 of the bytes — send it back as If-Match to write safely", body = String, content_type = "application/yaml"), (status = 304, description = "If-None-Match matched: unchanged"), @@ -473,7 +483,7 @@ pub async fn get_config_file( path = "/api/v1/config/{file}", tag = "Configuration", params( - ("file" = String, Path, description = "File name, e.g. sources.yaml"), + ("file" = String, Path, description = "Path inside the configuration directory, e.g. sources/acme/src-d42-acme.yaml (slashes allowed)"), ("reload" = Option, Query, description = "Apply to the running process as part of the write") ), request_body(content = String, description = "The file's new contents, verbatim YAML", content_type = "application/yaml"), @@ -514,7 +524,7 @@ pub async fn put_config_file( path = "/api/v1/config/{file}", tag = "Configuration", params( - ("file" = String, Path, description = "File name, e.g. enrichers.yaml"), + ("file" = String, Path, description = "Path inside the configuration directory, e.g. sources/acme/src-d42-acme.yaml (slashes allowed)"), ("reload" = Option, Query, description = "Apply to the running process as part of the delete") ), responses( @@ -728,6 +738,9 @@ async fn write( None }; + // Named before the commit: afterwards, what a prune removed is no longer + // there to be listed. + let deleted = deleted_names(state, store, &change)?; store.commit(&change).map_err(|e| { metrics::counter!("unified_api_config_writes_total", "outcome" => "error").increment(1); ApiError::internal(format!("configuration written to nowhere: {}", e)) @@ -735,7 +748,6 @@ async fn write( metrics::counter!("unified_api_config_writes_total", "outcome" => "success").increment(1); let written: Vec = change.write.keys().cloned().collect(); - let deleted = deleted_names(state, store, &change)?; let reloaded = match resolved { Some(resolved) => { @@ -786,8 +798,8 @@ fn changed_names(change: &ConfigChange) -> Vec { names } -// Which files the commit actually removed: the explicit deletions, plus -// whatever a pruning push left out. +// Which files the commit will remove: the explicit deletions, plus whatever +// a pruning push leaves out. fn deleted_names( _state: &Arc, store: &Arc, @@ -796,9 +808,9 @@ fn deleted_names( let present = store.stat_all().map_err(ApiError::internal)?; let mut deleted: Vec = change.delete.iter().cloned().collect(); if change.prune { - for name in CONFIG_FILES { - if !change.write.contains_key(name) && !present.iter().any(|s| s.name == name) { - deleted.push(name.to_string()); + for stat in &present { + if !change.write.contains_key(&stat.name) { + deleted.push(stat.name.clone()); } } } diff --git a/src/adapters/in/http/connectors.rs b/src/adapters/in/http/connectors.rs new file mode 100644 index 0000000..59597fa --- /dev/null +++ b/src/adapters/in/http/connectors.rs @@ -0,0 +1,212 @@ +use axum::Json; +use axum::extract::{Path, State}; +use serde::Serialize; +use std::sync::Arc; +use utoipa::ToSchema; + +use crate::AppState; +use crate::adapters::r#in::http::auth::AuthContext; +use crate::adapters::r#in::http::error::{ApiError, ErrorBody}; +use crate::domain::connector::{self, Connector}; +use crate::state::RuntimeConfig; + +// Connectors: HOW this instance talks to each kind of system, and which +// sources use each one. Admin-only, like projects: a connector names the +// script, its project and its arguments for every source of that system, +// which is deploy tooling rather than consumer data. +// +// The two built-ins (`remote`, `static_inventory`) are listed beside the +// connector items so a source's `connector:` always resolves to a row here. + +#[derive(Serialize, ToSchema)] +pub struct ConnectorInfo { + /// The connector's identity: what a source writes after `connector:` + pub connector_id: String, + /// Its description, or the identity when it has none + pub name: String, + /// "script", "ssh", "remote" or "static_inventory" + pub connector_type: String, + /// One of the connectors every instance has, which no item defines + pub built_in: bool, + /// The project whose checkout holds the script. Null for ssh (the script + /// runs on the remote host) and for the built-ins (a static_inventory + /// source names its own project) + pub project_id: Option, + /// Null for the built-ins + pub path: Option, + pub script_args: Vec, + pub host_args: Vec, + /// "native" or "ansible" for a script connector; null otherwise + pub output_format: Option, + /// The connector's own bound on a sync; a source may override it. Null for + /// the built-ins, whose sources fall back to 300 + pub timeout_seconds: Option, + /// The sources that name this connector, by id + pub used_by: Vec, + /// Those sources' sync health, counted + pub health: ConnectorHealth, +} + +#[derive(Serialize, ToSchema)] +pub struct ConnectorUser { + pub source_id: String, + pub name: String, + /// The timeout this source actually syncs under (its override, or the + /// connector's) + pub timeout_seconds: u64, + /// Whether the last sync of this source failed + pub failing: bool, + /// Whether it has synced successfully at least once in this process + pub synced: bool, +} + +/// What a sync-health summary across a connector's sources says at a glance. +#[derive(Serialize, ToSchema, Default)] +pub struct ConnectorHealth { + pub sources: usize, + /// Last attempt succeeded + pub healthy: usize, + /// Last attempt failed (consecutive_failures > 0) + pub failing: usize, + /// No attempt recorded yet in this process + pub never_synced: usize, +} + +fn info( + state: &AppState, + config: &RuntimeConfig, + id: &str, + connector: Option<&Connector>, +) -> ConnectorInfo { + let mut used_by: Vec = config + .sources + .iter() + .filter(|(_, s)| s.connector == id) + .map(|(source_id, s)| { + let health = state.sync_health.get(source_id); + ConnectorUser { + source_id: source_id.clone(), + name: s.name.clone(), + timeout_seconds: s.timeout_seconds, + failing: health.as_ref().is_some_and(|h| h.consecutive_failures > 0), + synced: health + .as_ref() + .is_some_and(|h| h.last_success_age_seconds.is_some()), + } + }) + .collect(); + used_by.sort_by(|a, b| a.source_id.cmp(&b.source_id)); + + let mut health = ConnectorHealth { + sources: used_by.len(), + ..ConnectorHealth::default() + }; + for user in &used_by { + match state.sync_health.get(&user.source_id) { + None => health.never_synced += 1, + Some(h) if h.consecutive_failures > 0 => health.failing += 1, + Some(_) => health.healthy += 1, + } + } + + match connector { + Some(c) => ConnectorInfo { + connector_id: id.to_string(), + name: c.name.clone(), + connector_type: c.kind.name().to_string(), + built_in: false, + project_id: c.project.clone(), + path: Some(c.path.clone()), + script_args: c.script_args.clone(), + host_args: c.host_args.clone(), + output_format: match c.kind { + connector::ConnectorKind::Script => { + Some(c.output_format.unwrap_or_default().name().to_string()) + } + connector::ConnectorKind::Ssh => None, + }, + timeout_seconds: Some(c.timeout_seconds), + used_by, + health, + }, + None => ConnectorInfo { + connector_id: id.to_string(), + name: match id { + connector::REMOTE => "Another unified-api instance".to_string(), + _ => "Ansible YAML inventory in a project checkout".to_string(), + }, + connector_type: id.to_string(), + built_in: true, + project_id: None, + path: None, + script_args: Vec::new(), + host_args: Vec::new(), + output_format: None, + timeout_seconds: None, + used_by, + health, + }, + } +} + +#[utoipa::path( + get, + path = "/api/v1/connectors", + tag = "Connectors", + responses( + (status = 200, description = "Every connector (the configured ones and the two built-ins), what it runs, the sources that use it and their health", body = Vec), + (status = 403, description = "API key is not admin", body = ErrorBody) + ) +)] +pub async fn list_connectors( + State(state): State>, + axum::Extension(auth): axum::Extension, +) -> Result>, ApiError> { + if !auth.permissions.is_admin() { + return Err(ApiError::admin_only()); + } + let config = state.config(); + let mut out: Vec = config + .connectors + .iter() + .map(|(id, c)| info(&state, &config, id, Some(c))) + .collect(); + out.sort_by(|a, b| a.connector_id.cmp(&b.connector_id)); + // Built-ins last: they are the same on every instance, and what an + // operator came to look at is usually one of their own. + for id in connector::BUILT_IN { + out.push(info(&state, &config, id, None)); + } + Ok(Json(out)) +} + +#[utoipa::path( + get, + path = "/api/v1/connectors/{id}", + tag = "Connectors", + params(("id" = String, Path, description = "Connector identifier, e.g. d42, or a built-in: remote, static_inventory")), + responses( + (status = 200, description = "One connector, what it runs, the sources that use it and their health", body = ConnectorInfo), + (status = 403, description = "API key is not admin", body = ErrorBody), + (status = 404, description = "No such connector", body = ErrorBody) + ) +)] +pub async fn get_connector( + State(state): State>, + axum::Extension(auth): axum::Extension, + Path(id): Path, +) -> Result, ApiError> { + if !auth.permissions.is_admin() { + return Err(ApiError::admin_only()); + } + let config = state.config(); + if let Some(c) = config.connectors.get(&id) { + return Ok(Json(info(&state, &config, &id, Some(c)))); + } + if connector::BUILT_IN.contains(&id.as_str()) { + return Ok(Json(info(&state, &config, &id, None))); + } + Err(ApiError::not_found(format!( + "connector '{id}' is not configured" + ))) +} diff --git a/src/adapters/in/http/detail.rs b/src/adapters/in/http/detail.rs index 6d8f456..2d764f5 100644 --- a/src/adapters/in/http/detail.rs +++ b/src/adapters/in/http/detail.rs @@ -68,8 +68,14 @@ pub struct SourceDetail { #[derive(Serialize, ToSchema)] pub struct Gathering { + /// The connector this source names: a connector item + /// (`/api/v1/connectors/{id}`) or a built-in (`remote`, `static_inventory`) + pub connector: String, pub connector_type: ConnectorType, - pub project_id: String, + /// The project whose checkout holds the script (from the connector) or + /// the inventory (a static_inventory source's own). Null for remote and + /// ssh sources, which run nothing from a checkout + pub project_id: Option, pub script_path: String, pub sync_mode: SyncMode, pub timeout_seconds: u64, @@ -142,6 +148,7 @@ pub async fn get_source( dataset_bytes: entry.as_ref().and_then(|e| e.serialized_len()), sync_health: state.sync_health.get(&id).map(Into::into), gathering: auth.permissions.is_admin().then(|| Gathering { + connector: source.connector.clone(), connector_type: source.connector_type.clone(), project_id: source.project_id.clone(), script_path: source.script_path.clone(), diff --git a/src/adapters/in/http/html.rs b/src/adapters/in/http/html.rs index ef74159..6597160 100644 --- a/src/adapters/in/http/html.rs +++ b/src/adapters/in/http/html.rs @@ -468,7 +468,7 @@ fn action_bar(ctx: &PageContext, template: &str, value: &Value) -> String { let target = format!("{}{}", ctx.path, action.suffix); let label = match (action.method, action.suffix.as_str()) { ("DELETE", "") => "Evict cache".to_string(), - ("POST", "/sync") => "Sync now".to_string(), + ("POST", "/sync") => "Sync".to_string(), ("POST", suffix) => { // Title-cased by character, not by byte: `name[..1]` panics on // an empty suffix and on any multi-byte first character, and @@ -991,7 +991,7 @@ mod tests { } }); let html = page(&c, &source); - assert!(html.contains("Sync now")); + assert!(html.contains(">Sync<")); assert!(html.contains("Evict cache")); } diff --git a/src/adapters/in/http/index.rs b/src/adapters/in/http/index.rs index e7627cd..0a40285 100644 --- a/src/adapters/in/http/index.rs +++ b/src/adapters/in/http/index.rs @@ -78,6 +78,9 @@ pub struct ApiIndex { /// Admin-only route; absent for a restricted key #[serde(skip_serializing_if = "Option::is_none")] pub projects: Option<&'static str>, + /// Admin-only route; absent for a restricted key + #[serde(skip_serializing_if = "Option::is_none")] + pub connectors: Option<&'static str>, /// Admin-only, and only when config_api.enabled is on #[serde(skip_serializing_if = "Option::is_none")] pub config: Option<&'static str>, @@ -119,6 +122,7 @@ pub async fn api_index( enrichers: "/api/v1/enrichers", endpoints: "/api/v1/endpoints", projects: admin.then_some("/api/v1/projects"), + connectors: admin.then_some("/api/v1/connectors"), // Same condition the config handlers enforce: the routes exist in the // router either way, and answer 403 when no store is configured. config: (admin && state.config_store.is_some()).then_some("/api/v1/config"), diff --git a/src/adapters/in/http/login.rs b/src/adapters/in/http/login.rs index a34072c..23290d4 100644 --- a/src/adapters/in/http/login.rs +++ b/src/adapters/in/http/login.rs @@ -69,7 +69,7 @@ fn safe_next<'a>(next: Option<&'a str>, home: &'a str) -> &'a str { // For a `next` carried in a query string: every byte but the unreserved ones // and '/' escaped, so a path with its own query ("?host=a&b") arrives whole. -fn query_escape(value: &str) -> String { +pub(crate) fn query_escape(value: &str) -> String { value .bytes() .map(|b| match b { @@ -372,7 +372,7 @@ pub async fn login_submit( ); } - let (name, kind, permissions) = if let Some(username) = form.username.as_deref() { + let (name, kind, key_id, permissions) = if let Some(username) = form.username.as_deref() { let Some(ldap) = ldap.as_ref() else { return refuse("LDAP login is not configured", StatusCode::BAD_REQUEST); }; @@ -390,6 +390,7 @@ pub async fn login_submit( Ok(logged) => ( format!("ldap:{}", logged.username), SessionKind::Directory, + None, match logged.grant { Grant::Admin => Permissions::Admin, Grant::Scoped { sources, endpoints } => { @@ -455,11 +456,14 @@ pub async fn login_submit( ( key.name.clone(), SessionKind::ApiKey, + Some(key.id.clone()), key.permissions.clone(), ) }; - let issued = sessions.0.create(name.clone(), kind, permissions, ttl.0); + let issued = sessions + .0 + .create(name.clone(), kind, key_id, permissions, ttl.0); tracing::info!(key_name = %name, "browser session opened"); diff --git a/src/adapters/in/http/mod.rs b/src/adapters/in/http/mod.rs index a8ecbba..a33e7d1 100644 --- a/src/adapters/in/http/mod.rs +++ b/src/adapters/in/http/mod.rs @@ -4,6 +4,7 @@ pub mod audit; pub mod auth; pub mod cache; pub mod config; +pub mod connectors; pub mod credentials; pub mod detail; pub mod endpoints; diff --git a/src/adapters/in/http/openapi.rs b/src/adapters/in/http/openapi.rs index d5816fa..f9d99fa 100644 --- a/src/adapters/in/http/openapi.rs +++ b/src/adapters/in/http/openapi.rs @@ -35,6 +35,7 @@ impl Modify for SecurityAddon { http::detail::get_source, http::sources::get_source_dataset, http::sources::list_source_groups, + http::sources::get_source_group, http::sources::list_source_hosts, http::sources::source_status, http::scope::source_scope, @@ -52,8 +53,12 @@ impl Modify for SecurityAddon { http::host_search::search_hosts, http::endpoints::list_endpoints, http::projects::list_projects, + http::connectors::list_connectors, + http::connectors::get_connector, http::users::get_auth, http::topology::get_topology, + http::topology::get_federation, + http::topology::put_federation, http::projects::sync_project_now, http::config::get_config, http::config::get_config_file, @@ -70,6 +75,7 @@ impl Modify for SecurityAddon { http::detail::SourceDetail, http::detail::Gathering, http::sources::GroupInfo, + http::sources::GroupDetail, http::sources::HostList, http::sources::GroupsMode, http::sources::HostStatus, @@ -88,6 +94,9 @@ impl Modify for SecurityAddon { http::health::ReadyStatus, http::projects::ProjectInfo, http::projects::ProjectUsers, + http::connectors::ConnectorInfo, + http::connectors::ConnectorUser, + http::connectors::ConnectorHealth, http::credentials::CredentialInfo, http::credentials::FileKeyInfo, http::credentials::CredentialUsersInfo, @@ -106,6 +115,9 @@ impl Modify for SecurityAddon { crate::domain::topology::Topology, crate::domain::topology::Peer, crate::domain::topology::PeerSource, + crate::domain::topology::FederationPush, + crate::domain::topology::Federation, + crate::domain::topology::InstanceSummary, http::users::Me, http::users::Methods, http::users::LdapInfo, @@ -128,6 +140,7 @@ impl Modify for SecurityAddon { (name = "Enrichers", description = "Post-processing enrichment of cached data"), (name = "Endpoints", description = "Output endpoints for consumers (AWX, AnsibleForms)"), (name = "Projects", description = "Git project checkouts (admin-only operational routes)"), + (name = "Connectors", description = "How this instance talks to each kind of system — the script, its project and arguments — and which sources use each connector (admin-only)"), (name = "Credentials", description = "Configured credentials: where each one's secrets come from, whether they resolve now, and what uses it (admin-only; never a secret value)"), (name = "Topology", description = "The federation seen from this instance: the unified-api instances it pulls from, and theirs"), (name = "Auth", description = "Who the caller is, and the ways in: LDAP login and API keys (read-only; declared in git)"), diff --git a/src/adapters/in/http/projects.rs b/src/adapters/in/http/projects.rs index 425f4b2..3e99956 100644 --- a/src/adapters/in/http/projects.rs +++ b/src/adapters/in/http/projects.rs @@ -22,8 +22,12 @@ pub struct ProjectInfo { pub name: String, pub git_url: String, pub branch: String, + /// The credential it pulls with, when the repository needs one + pub credential_id: Option, /// Whether a checkout currently exists on disk pub checkout_present: bool, + /// The commit the checkout is at (full id); null without a checkout + pub revision: Option, /// Seconds between periodic re-pulls (absent/0 = only boot and on demand) pub sync_interval_seconds: Option, pub sync_on_boot: bool, @@ -46,6 +50,10 @@ pub struct ProjectInfo { #[derive(Serialize, ToSchema)] pub struct ProjectUsers { + /// Connectors whose scripts live in this checkout + pub connectors: Vec, + /// Sources that run from it: through their connector, or (static + /// inventory) by naming it themselves pub sources: Vec, pub endpoints: Vec, pub enrichers: Vec, @@ -106,13 +114,20 @@ pub async fn list_projects( // clones from the configured value. git_url: crate::domain::topology::without_userinfo(&project.git_url), branch: project.branch.clone(), + credential_id: project.credential_id.clone(), checkout_present, + revision: if checkout_present { + state.git.revision(&state.projects_dir.join(id)).await + } else { + None + }, sync_interval_seconds: project.sync_interval_seconds, sync_on_boot: project.sync_on_boot, schedule: project.schedule.clone(), timeout_seconds: project.timeout_seconds, used_by: ProjectUsers { - sources: users_of(&config.sources, id, |s| Some(s.project_id.as_str())), + connectors: users_of(&config.connectors, id, |c| c.project.as_deref()), + sources: users_of(&config.sources, id, |s| s.project_id.as_deref()), endpoints: users_of(&config.endpoints, id, |e| e.project_id.as_deref()), enrichers: users_of(&config.enrichers, id, |e| e.project_id.as_deref()), }, diff --git a/src/adapters/in/http/routes.rs b/src/adapters/in/http/routes.rs index 13edc28..362c334 100644 --- a/src/adapters/in/http/routes.rs +++ b/src/adapters/in/http/routes.rs @@ -50,6 +50,10 @@ pub fn create_router(state: Arc, api_keys: Arc) -> Rou "/api/v1/sources/{id}/groups", get(http::sources::list_source_groups), ) + .route( + "/api/v1/sources/{id}/groups/{group}", + get(http::sources::get_source_group), + ) .route( "/api/v1/sources/{id}/hosts", get(http::sources::list_source_hosts), @@ -79,6 +83,11 @@ pub fn create_router(state: Arc, api_keys: Arc) -> Rou get(http::endpoints::endpoint_counts), ) .route("/api/v1/projects", get(http::projects::list_projects)) + .route("/api/v1/connectors", get(http::connectors::list_connectors)) + .route( + "/api/v1/connectors/{id}", + get(http::connectors::get_connector), + ) .route("/api/v1/settings", get(http::settings::get_settings)) .route("/api/v1/hosts/search", get(http::host_search::search_hosts)) .route( @@ -87,6 +96,10 @@ pub fn create_router(state: Arc, api_keys: Arc) -> Rou ) .route("/api/v1/auth", get(http::users::get_auth)) .route("/api/v1/topology", get(http::topology::get_topology)) + .route( + "/api/v1/topology/federation", + get(http::topology::get_federation).put(http::topology::put_federation), + ) .route( "/api/v1/projects/{id}/sync", post(http::projects::sync_project_now), @@ -104,8 +117,10 @@ pub fn create_router(state: Arc, api_keys: Arc) -> Rou post(http::config::validate_config), ) .route("/api/v1/config/reload", post(http::config::reload_config)) + // `{*file}`: a configuration file may sit in folders, so the capture + // takes the rest of the path, slashes included. .route( - "/api/v1/config/{file}", + "/api/v1/config/{*file}", get(http::config::get_config_file) .put(http::config::put_config_file) .delete(http::config::delete_config_file), @@ -151,11 +166,9 @@ pub fn create_router(state: Arc, api_keys: Arc) -> Rou // The built UI, when this instance serves one (server.ui.assets_dir, and // only with the browsable UI on: it signs in through the login page). - let ui_assets = ui - .assets_dir - .as_ref() - .filter(|_| ui.enabled) - .map(http::ui_assets::UiAssets::new); + let ui_assets = ui.assets_dir.as_ref().filter(|_| ui.enabled).map(|dir| { + http::ui_assets::UiAssets::new(dir).with_gate(Arc::clone(&api_keys), Arc::clone(&sessions)) + }); let metrics_route = match &ui_assets { Some(assets) => metrics_route.layer(middleware::from_fn_with_state( assets.clone(), diff --git a/src/adapters/in/http/session.rs b/src/adapters/in/http/session.rs index b07fc46..fd11258 100644 --- a/src/adapters/in/http/session.rs +++ b/src/adapters/in/http/session.rs @@ -39,6 +39,11 @@ pub struct Session { // access log exactly as a header-authenticated request is. pub key_name: String, pub kind: SessionKind, + // The key's identity (its api_keys `name`) for a key session: what a + // reload revokes it by. Its label (key_name, the description) is for + // people and need not be unique — two keys described alike, one removed, + // and revoking by label kept the removed key's sessions working. + key_id: Option, pub permissions: Permissions, // Paired with the session and required on every unsafe request that // authenticates by cookie. See `auth::require_api_key` for why. @@ -72,6 +77,7 @@ impl SessionStore { &self, key_name: String, kind: SessionKind, + key_id: Option, permissions: Permissions, ttl: Duration, ) -> Issued { @@ -90,6 +96,7 @@ impl SessionStore { Session { key_name, kind, + key_id, permissions, csrf: csrf.clone(), expires_at: Instant::now() + ttl, @@ -130,9 +137,11 @@ impl SessionStore { // revoked by that reload keeps working in every browser that logged in // before it — the console would outlive the revocation, which is exactly // the property a revocation is for. + // `live` holds key identities (ResolvedApiKey::id), never labels. pub fn retain_keys(&self, live: &[String]) { self.sessions.retain(|_, session| { - session.kind != SessionKind::ApiKey || live.contains(&session.key_name) + session.kind != SessionKind::ApiKey + || session.key_id.as_ref().is_some_and(|id| live.contains(id)) }); } @@ -356,6 +365,7 @@ mod tests { let issued = store.create( "AWX".into(), SessionKind::ApiKey, + Some("AWX".into()), admin(), Duration::from_secs(60), ); @@ -376,6 +386,7 @@ mod tests { let issued = store.create( "AWX".into(), SessionKind::ApiKey, + Some("AWX".into()), admin(), Duration::from_secs(0), ); @@ -389,6 +400,7 @@ mod tests { let issued = store.create( "AWX".into(), SessionKind::ApiKey, + Some("AWX".into()), admin(), Duration::from_secs(60), ); @@ -404,12 +416,14 @@ mod tests { let stays = store.create( "AWX".into(), SessionKind::ApiKey, + Some("AWX".into()), admin(), Duration::from_secs(60), ); let goes = store.create( "Retired".into(), SessionKind::ApiKey, + Some("Retired".into()), admin(), Duration::from_secs(60), ); @@ -420,6 +434,32 @@ mod tests { assert!(store.get(&goes.token).is_none()); } + // Revoked by identity, not by label: two keys described alike, and the + // one removed takes only its own sessions with it. + #[test] + fn a_key_is_revoked_by_its_identity_not_its_label() { + let store = SessionStore::new(); + let stays = store.create( + "Automation".into(), + SessionKind::ApiKey, + Some("key-awx".into()), + admin(), + Duration::from_secs(60), + ); + let goes = store.create( + "Automation".into(), + SessionKind::ApiKey, + Some("key-old".into()), + admin(), + Duration::from_secs(60), + ); + + store.retain_keys(&["key-awx".to_string()]); + + assert!(store.get(&stays.token).is_some()); + assert!(store.get(&goes.token).is_none()); + } + // An LDAP session is not a key: a key reload must leave it alone, and an // LDAP change must end it without touching the key sessions. #[test] @@ -428,12 +468,14 @@ mod tests { let key = store.create( "AWX".into(), SessionKind::ApiKey, + Some("AWX".into()), admin(), Duration::from_secs(60), ); let person = store.create( "ldap:jdoe".into(), SessionKind::Directory, + None, admin(), Duration::from_secs(60), ); @@ -455,12 +497,14 @@ mod tests { let a = store.create( "k".into(), SessionKind::ApiKey, + Some("k".into()), admin(), Duration::from_secs(60), ); let b = store.create( "k".into(), SessionKind::ApiKey, + Some("k".into()), admin(), Duration::from_secs(60), ); diff --git a/src/adapters/in/http/settings.rs b/src/adapters/in/http/settings.rs index e048e83..8c38172 100644 --- a/src/adapters/in/http/settings.rs +++ b/src/adapters/in/http/settings.rs @@ -50,6 +50,10 @@ pub struct ServerSettings { pub refresh_max_concurrent: usize, pub shutdown_grace_seconds: u64, pub max_body_bytes: usize, + /// This instance's address, as `server.url` declares it; null when unset + pub url: Option, + /// What this instance is called, as `server.name` declares it + pub name: Option, pub ui: UiSettings, } @@ -144,6 +148,8 @@ pub async fn get_settings( refresh_max_concurrent: config.refresh_max_concurrent, shutdown_grace_seconds: config.shutdown_grace_seconds, max_body_bytes: config.max_body_bytes, + url: config.url.clone(), + name: config.name.clone(), ui: UiSettings { enabled: state.ui.enabled, session_ttl_seconds: state.ui.session_ttl_seconds, diff --git a/src/adapters/in/http/sources.rs b/src/adapters/in/http/sources.rs index 0b7ea63..625db4a 100644 --- a/src/adapters/in/http/sources.rs +++ b/src/adapters/in/http/sources.rs @@ -811,6 +811,70 @@ pub async fn list_source_groups( Ok(Json(groups)) } +#[derive(Serialize, ToSchema)] +pub struct GroupDetail { + pub name: String, + /// Unique hosts in the group + pub host_count: usize, + pub children: Vec, + /// The group's own variables, sorted by name; empty when it has none + #[schema(value_type = Object)] + pub vars: std::collections::BTreeMap, +} + +// One group: its variables, children and size, without the rest of the +// dataset. The variables are otherwise only in /dataset, which carries every +// host's facts too. +#[utoipa::path( + get, + path = "/api/v1/sources/{id}/groups/{group}", + tag = "Sources", + params( + ("id" = String, Path, description = "Source identifier"), + ("group" = String, Path, description = "Group name") + ), + responses( + (status = 200, description = "The group: its variables, its child groups and how many hosts it has", body = GroupDetail), + (status = 403, description = "API key not allowed to read this source", body = ErrorBody), + (status = 404, description = "Source not in cache, or no such group in it", body = ErrorBody) + ) +)] +pub async fn get_source_group( + State(state): State>, + Extension(auth): Extension, + Path((id, group)): Path<(String, String)>, +) -> Result, ApiError> { + if !auth.permissions.allows_source(&id) { + return Err(ApiError::source_forbidden(&id)); + } + let not_found = || ApiError::not_found(format!("source '{id}' has no group '{group}'")); + let config = state.config(); + if let Some(view) = config.views.get(&id) { + return views::group(&state, &id, view, &group) + .map(Json) + .ok_or_else(not_found); + } + let entry = state + .cache + .get(&id) + .ok_or_else(|| ApiError::source_not_cached(&id))?; + let found = entry.dataset.groups.get(&group).ok_or_else(not_found)?; + Ok(Json(GroupDetail { + name: group.clone(), + host_count: found + .hosts + .iter() + .collect::>() + .len(), + children: found.children.clone(), + vars: found + .vars + .as_ref() + .map(|v| v.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default(), + })) +} + #[derive(Serialize, ToSchema)] pub struct HostList { pub source_id: String, diff --git a/src/adapters/in/http/spec.rs b/src/adapters/in/http/spec.rs index 4f6c2e2..43c1905 100644 --- a/src/adapters/in/http/spec.rs +++ b/src/adapters/in/http/spec.rs @@ -43,17 +43,26 @@ fn document() -> &'static utoipa::openapi::OpenApi { DOC.get_or_init(ApiDoc::openapi) } +// Templates whose last capture takes the rest of the path, slashes included +// (`{*file}` in the router; OpenAPI has no such syntax, so the spec says +// `{file}` and this list says the rest). +const GREEDY: [&str; 1] = ["/api/v1/config/{file}"]; + // Does a concrete path match a route template? Segment by segment, with // `{name}` matching exactly one segment — the same rule axum's router uses, so // the template this finds is the template that handled the request. fn matches(template: &str, path: &str) -> bool { let template = template.trim_end_matches('/'); let path = path.trim_end_matches('/'); - let mut t = template.split('/'); + let greedy = GREEDY.contains(&template); + let mut t = template.split('/').peekable(); let mut p = path.split('/'); loop { match (t.next(), p.next()) { (None, None) => return true, + (Some(t_seg), Some(p_seg)) if greedy && t.peek().is_none() => { + return !p_seg.is_empty() && t_seg.starts_with('{'); + } (Some(t), Some(p)) => { let wildcard = t.starts_with('{') && t.ends_with('}'); if !wildcard && t != p { @@ -198,6 +207,15 @@ mod tests { "/api/v1/sources/src-d42/hosts/web01" )); assert!(matches("/api/v1/sources", "/api/v1/sources")); + // A configuration file can sit in folders. + assert!(matches( + "/api/v1/config/{file}", + "/api/v1/config/sources/acme/src-d42-acme.yaml" + )); + assert!(!matches( + "/api/v1/sources/{id}", + "/api/v1/sources/src-d42/sync" + )); } #[test] diff --git a/src/adapters/in/http/topology.rs b/src/adapters/in/http/topology.rs index 17e360b..56305f6 100644 --- a/src/adapters/in/http/topology.rs +++ b/src/adapters/in/http/topology.rs @@ -5,11 +5,11 @@ use std::sync::Arc; use utoipa::IntoParams; use crate::AppState; -use crate::adapters::r#in::http::auth::AuthContext; +use crate::adapters::r#in::http::auth::{AuthContext, AuthMethod}; use crate::adapters::r#in::http::error::ApiError; use crate::adapters::r#in::http::error::ErrorBody; use crate::application::topology::{Inputs, MAX_DEPTH, describe}; -use crate::domain::topology::Topology; +use crate::domain::topology::{Federation, FederationPush, Topology}; // Declared as query parameters here: the handler takes them wrapped in a // Result (to answer a malformed one as JSON), which hides the Query the @@ -53,6 +53,7 @@ pub async fn get_topology( health: state.sync_health.as_ref(), secrets: state.secrets.as_ref(), peers: Arc::clone(&state.peers), + name: config.name.as_deref(), }, |id| auth.permissions.allows_source(id), query.depth.unwrap_or(2).min(MAX_DEPTH), @@ -61,77 +62,72 @@ pub async fn get_topology( Ok(Json(if auth.permissions.is_admin() { topology } else { - for_restricted(topology) + topology.redacted() })) } -// What a restricted caller may see of the peers behind the sources it reads. -// The peers were asked with THIS instance's key, which may be an admin key -// over there: their own peers, URLs and counts are that key's view, not the -// caller's, so only each peer's version is kept. And why a peer was not -// asked stays generic: the resolver's message names env vars and Vault -// paths, which /credentials keeps for admins. -fn for_restricted(mut topology: Topology) -> Topology { - for peer in &mut topology.peers { - if let Some(instance) = peer.instance.as_mut() { - instance.sources = None; - instance.peers = Vec::new(); - } - if peer - .error - .as_deref() - .is_some_and(|e| e.starts_with("not asked:") && !e.contains("depth limit")) - { - peer.error = Some("not asked: this instance cannot authenticate to it".to_string()); +// An instance that reads from this one says what the federation looks like +// from there (application::topology::push_federation, every minute). +// +// Any API key may push: the instance above holds the restricted key its +// remote sources read with, and requiring admin would mean handing it more +// than it reads. What a push can do is bounded by that: it replaces the +// sender's own picture (stored under its key's identity, its `name`), shown +// to admins only, and changes nothing else. A browser session cannot push, +// whether it signed in through the directory or with a key: a session has no +// key identity to store the map under (its label is not unique, and could +// overwrite another key's entry), and no instance pushes through one. +#[utoipa::path( + put, + path = "/api/v1/topology/federation", + tag = "Topology", + request_body = FederationPush, + responses( + (status = 204, description = "Stored under the sending key's name, replacing that key's previous push"), + (status = 400, description = "The body is not a federation push", body = ErrorBody), + (status = 401, description = "Not authenticated", body = ErrorBody), + (status = 403, description = "A browser session (signed in through the UI, with a key or a directory account) rather than an API key sent with the request", body = ErrorBody) + ) +)] +pub async fn put_federation( + State(state): State>, + axum::Extension(auth): axum::Extension, + body: Result, axum::extract::rejection::JsonRejection>, +) -> Result { + // A key sent with the request is the only way to have a `key_id`; an + // open instance (no keys at all) has nobody to tell apart. + let sender = match (auth.method, auth.key_id) { + (AuthMethod::ApiKey, Some(id)) => id, + (AuthMethod::Open, _) => "open".to_string(), + _ => { + return Err(ApiError::forbidden( + "a federation map is pushed by another instance, with an API key", + )); } - } - topology + }; + let Json(push) = body.map_err(|e| ApiError::bad_request(e.body_text()))?; + state.federation.record(sender, push); + Ok(axum::http::StatusCode::NO_CONTENT) } -#[cfg(test)] -mod tests { - use super::*; - use crate::domain::topology::Peer; - - fn peer(instance: Option, error: Option<&str>) -> Peer { - Peer { - url: "https://edge.example".to_string(), - sources: Vec::new(), - instance: instance.map(Box::new), - error: error.map(str::to_string), - } - } - - #[test] - fn a_restricted_caller_sees_a_peers_version_and_no_secret_locations() { - let behind = Topology { - version: "0.33.0".to_string(), - sources: Some(12), - peers: vec![peer(None, None)], - }; - let topology = Topology { - version: "0.34.0".to_string(), - sources: Some(1), - peers: vec![ - peer(Some(behind), None), - peer(None, Some("not asked: Env var 'EDGE_TOKEN' not set")), - peer(None, Some("not asked: depth limit reached")), - ], - }; - let seen = for_restricted(topology); - let instance = seen.peers[0].instance.as_ref().unwrap(); - assert_eq!(instance.version, "0.33.0"); - assert!(instance.peers.is_empty() && instance.sources.is_none()); - assert!( - !seen.peers[1] - .error - .as_deref() - .unwrap() - .contains("EDGE_TOKEN") - ); - assert_eq!( - seen.peers[2].error.as_deref(), - Some("not asked: depth limit reached") - ); +// The maps pushed here, newest first: on an edge, the federation it belongs +// to. Admin-only: it is the sender's whole map, other instances included. +#[utoipa::path( + get, + path = "/api/v1/topology/federation", + tag = "Topology", + responses( + (status = 200, description = "One entry per instance that reads from this one and pushed its map in the last ten minutes. Empty on an instance nothing reads from", body = [Federation]), + (status = 401, description = "Not authenticated", body = ErrorBody), + (status = 403, description = "Not an admin key", body = ErrorBody) + ) +)] +pub async fn get_federation( + State(state): State>, + axum::Extension(auth): axum::Extension, +) -> Result>, ApiError> { + if !auth.permissions.is_admin() { + return Err(ApiError::admin_only()); } + Ok(Json(state.federation.list())) } diff --git a/src/adapters/in/http/ui_assets.rs b/src/adapters/in/http/ui_assets.rs index 3635f39..e335f04 100644 --- a/src/adapters/in/http/ui_assets.rs +++ b/src/adapters/in/http/ui_assets.rs @@ -11,6 +11,7 @@ // the app, or a client would read a 200 HTML page as an answer. use std::path::PathBuf; +use std::sync::Arc; use axum::body::Body; use axum::extract::{Request, State}; @@ -20,11 +21,18 @@ use axum::response::{IntoResponse, Response}; use tower::ServiceExt; use tower_http::services::{ServeDir, ServeFile}; +use crate::adapters::r#in::http::auth::ApiKeyRegistry; use crate::adapters::r#in::http::error::ApiError; +use crate::adapters::r#in::http::session::{COOKIE_NAME, SessionStore, cookie_value}; #[derive(Clone)] pub struct UiAssets { dir: PathBuf, + // Who may see the app's pages: a browser with no session is sent to the + // login form by the server, before the app loads, instead of loading the + // app only for it to find out it is signed out and redirect itself (a + // flash of the overview, then the login page). + gate: Option<(Arc, Arc)>, } impl UiAssets { @@ -38,7 +46,43 @@ impl UiAssets { "server.ui.assets_dir has no index.html: the UI will not load" ); } - Self { dir } + Self { dir, gate: None } + } + + pub fn with_gate(mut self, keys: Arc, sessions: Arc) -> Self { + self.gate = Some((keys, sessions)); + self + } + + // Where a browser asking for `request` must go first: the login form, + // when keys are configured (no keys = an open API, and so an open app) + // and the request carries no live session. Only page loads are gated: + // the app's own files (scripts, styles, icons) are not secrets, and the + // login page needs some of them. + fn sign_in_first(&self, request: &Request) -> Option { + let (keys, sessions) = self.gate.as_ref()?; + if keys.load().is_empty() || !is_page_load(request) { + return None; + } + let signed_in = request + .headers() + .get(header::COOKIE) + .and_then(|v| v.to_str().ok()) + .and_then(|c| cookie_value(c, COOKIE_NAME)) + .is_some_and(|token| sessions.get(token).is_some()); + if signed_in { + return None; + } + let next = request + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_string()) + .unwrap_or_else(|| "/".to_string()); + let login = format!( + "/api/v1/login?next={}", + crate::adapters::r#in::http::login::query_escape(&next) + ); + Some(axum::response::Redirect::to(&login).into_response()) } fn index(&self) -> PathBuf { @@ -46,6 +90,16 @@ impl UiAssets { } } +// A browser navigating to a page, as opposed to fetching one of the app's +// files: it asks for HTML. The app's scripts and styles ask for something else. +fn is_page_load(request: &Request) -> bool { + request + .headers() + .get(header::ACCEPT) + .and_then(|v| v.to_str().ok()) + .is_some_and(|accept| accept.contains("text/html")) +} + // Paths that belong to the API: a miss there is the API's 404, not the app. fn is_api_path(path: &str) -> bool { ["/api", "/swagger-ui", "/api-docs"] @@ -66,6 +120,9 @@ pub async fn serve(State(assets): State, request: Request) -> Response .into_response(); } + if let Some(redirect) = assets.sign_in_first(&request) { + return redirect; + } let service = ServeDir::new(&assets.dir).fallback(ServeFile::new(assets.index())); let response = match service.oneshot(request).await { Ok(response) => response.map(Body::new), @@ -106,6 +163,9 @@ pub async fn page_for_browsers( if !wants_html { return next.run(request).await; } + if let Some(redirect) = assets.sign_in_first(&request) { + return redirect; + } let response = match ServeFile::new(assets.index()).oneshot(request).await { Ok(response) => response.map(Body::new), Err(never) => match never {}, diff --git a/src/adapters/in/http/views.rs b/src/adapters/in/http/views.rs index 45086a0..62736c8 100644 --- a/src/adapters/in/http/views.rs +++ b/src/adapters/in/http/views.rs @@ -407,6 +407,35 @@ pub fn groups(state: &Arc, id: &str, view: &View) -> Vec { .collect() } +// GET /api/v1/sources/{id}/groups/{group} for a view: the group as the +// members' merge has it. None when no member has it. +pub fn group( + state: &Arc, + id: &str, + view: &View, + name: &str, +) -> Option { + let config = state.config(); + let snap = snapshot( + &*state.cache, + &config.sources, + &state.advertised_scopes, + id, + view, + ); + let groups = snap.groups(); + let group = groups.get(name)?; + Some(super::sources::GroupDetail { + name: name.to_string(), + host_count: group.hosts.len(), + children: group.children.iter().map(|c| c.to_string()).collect(), + vars: group + .vars + .map(|v| v.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default(), + }) +} + // GET /api/v1/sources/{id}/hosts for a view. pub fn hosts( state: &Arc, diff --git a/src/adapters/in/scheduler/mod.rs b/src/adapters/in/scheduler/mod.rs index e799995..807a5cb 100644 --- a/src/adapters/in/scheduler/mod.rs +++ b/src/adapters/in/scheduler/mod.rs @@ -380,6 +380,7 @@ pub fn start_supervisor( let mut handles = start_sync_tasks(Arc::clone(&state), stop_rx.clone()); handles.extend(start_project_tasks(Arc::clone(&state), stop_rx.clone())); + handles.extend(start_federation_task(Arc::clone(&state), stop_rx.clone())); // A project that arrived WITH this reload has no checkout yet, and // the scripts of any source pointing into it would be missing // until its first periodic pull — which for a project without an @@ -410,6 +411,81 @@ pub fn start_supervisor( }) } +// How often the federation map is pushed down to the peers. Each edge shows +// it with its age, and drops it after ten missed pushes. +const FEDERATION_PUSH_SECONDS: u64 = 60; + +// Tell every peer where it sits in the federation (application::topology:: +// push_federation). Only on an instance that reads from another one: with no +// remote sources there is nobody to tell, and no task. Part of the generation +// like the others, so a reload that adds or removes the last remote source +// starts or ends it. +fn start_federation_task( + state: Arc, + mut shutdown: watch::Receiver, +) -> Option> { + let config = state.config(); + let has_peers = config + .sources + .values() + .any(|s| crate::domain::topology::remote_url(s).is_some()); + if !has_peers { + return None; + } + Some(tokio::spawn(async move { + // Logged when a push starts or stops failing, not every minute. + let mut failing: std::collections::HashSet = Default::default(); + // After the first syncs have had a moment, so the first map pushed + // already says how the pulls are going. + let mut ticker = interval(Duration::from_secs(FEDERATION_PUSH_SECONDS)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + ticker.reset_after(Duration::from_secs(15)); + loop { + tokio::select! { + _ = ticker.tick() => {} + _ = shutdown.changed() => return, + } + let config = state.config(); + let push = crate::application::topology::push_federation( + crate::application::topology::Inputs { + sources: &config.sources, + cache: state.cache.as_ref(), + health: state.sync_health.as_ref(), + secrets: state.secrets.as_ref(), + peers: Arc::clone(&state.peers), + name: config.name.as_deref(), + }, + config.url.as_deref(), + config.endpoints.len(), + ); + // Cut short at shutdown, unlike a sync: it only reads here and + // writes a picture there, and waiting on a dead peer's timeouts + // would spend the drain's grace period on nothing. + let outcomes = tokio::select! { + outcomes = push => outcomes, + _ = shutdown.changed() => return, + }; + for (url, outcome) in outcomes { + use crate::ports::peer::PushError; + match outcome { + Ok(()) | Err(PushError::Unsupported) => { + if failing.remove(&url) { + info!(peer = %url, "Federation map reaches the peer again"); + } + } + Err(PushError::Failed(reason)) => { + if failing.insert(url.clone()) { + warn!(peer = %url, error = %reason, "Could not push the federation map"); + } else { + debug!(peer = %url, error = %reason, "Federation push still failing"); + } + } + } + } + } + })) +} + // The project tasks, wired from the state rather than from main's locals — // which is what lets a reload restart them with a different projects.yaml. fn start_project_tasks( @@ -1063,8 +1139,8 @@ mod tests { async fn a_cron_scheduled_enricher_runs_and_records_health() { let enricher: crate::domain::enricher::Enricher = serde_saphyr::from_str(concat!( "name: cron-merge\n", - "target_id: src-target\n", - "source_id: src-extra\n", + "target: src-target\n", + "source: src-extra\n", "fields: [\"role\"]\n", "schedule: \"* * * * * *\"\n", )) @@ -1196,7 +1272,7 @@ mod tests { #[tokio::test] async fn a_scheduled_enricher_runs_and_records_health() { let enricher: crate::domain::enricher::Enricher = serde_saphyr::from_str( - "name: e\ntarget_id: src-t\nsource_id: src-s\nfields: [\"f\"]\nsync_interval_seconds: 1\n", + "name: e\ntarget: src-t\nsource: src-s\nfields: [\"f\"]\nsync_interval_seconds: 1\n", ) .expect("enricher fixture"); let mut enrichers = HashMap::new(); @@ -1404,6 +1480,7 @@ mod tests { let previous = state.config(); state.swap_config(Arc::new(crate::RuntimeConfig { sources: after, + connectors: previous.connectors.clone(), credentials: previous.credentials.clone(), views: previous.views.clone(), enrichers: previous.enrichers.clone(), @@ -1419,6 +1496,8 @@ mod tests { refresh_max_concurrent: previous.refresh_max_concurrent, ldap: previous.ldap.clone(), ui_home: previous.ui_home.clone(), + url: previous.url.clone(), + name: previous.name.clone(), })); state.reload.bump(); diff --git a/src/adapters/out/config/fs.rs b/src/adapters/out/config/fs.rs index a942a0d..4d25196 100644 --- a/src/adapters/out/config/fs.rs +++ b/src/adapters/out/config/fs.rs @@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use sha2::{Digest, Sha256}; use crate::config::{ - AppConfig, CONFIG_FILES, ConfigErrors, REQUIRED_CONFIG_FILE, load_config_detailed, + AppConfig, ConfigErrors, REQUIRED_CONFIG_FILE, config_files, load_config_detailed, }; use crate::ports::config_store::{ConfigChange, ConfigFileStat, ConfigStorePort}; @@ -45,35 +45,48 @@ impl FsConfigStore { &self.dir } - // The contents every known file WOULD have with `change` applied: what is - // on disk, plus the writes, minus the deletions (and minus everything the - // payload does not name, when pruning). + // The contents every file WOULD have with `change` applied: what is on + // disk, plus the writes, minus the deletions (and minus everything the + // payload does not name, when pruning). Paths are relative to the + // directory, with `/`, at any depth. fn effective(&self, change: &ConfigChange) -> Result, String> { let mut files = BTreeMap::new(); - for name in CONFIG_FILES { - if change.prune && !change.write.contains_key(name) { - continue; - } - if change.delete.contains(name) { - continue; - } - if let Some(contents) = change.write.get(name) { - files.insert(name.to_string(), contents.clone()); - continue; - } - match std::fs::read_to_string(self.dir.join(name)) { - Ok(contents) => { - files.insert(name.to_string(), contents); + if !change.prune { + for name in config_files(&self.dir)? { + if change.delete.contains(&name) || change.write.contains_key(&name) { + continue; } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(format!("read '{}': {}", name, e)), + match std::fs::read_to_string(self.dir.join(&name)) { + Ok(contents) => { + files.insert(name, contents); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(format!("read '{}': {}", name, e)), + } + } + } + for (name, contents) in &change.write { + if !change.delete.contains(name) { + files.insert(name.clone(), contents.clone()); } } Ok(files) } + // After files are removed, the folders they leave empty go too, up to + // (never including) the configuration directory itself: an empty folder + // is not configuration, and a pruned push should leave exactly its tree. + fn remove_empty_parents(&self, name: &str) { + let mut dir = self.dir.join(name); + while dir.pop() && dir != self.dir && dir.starts_with(&self.dir) { + if std::fs::remove_dir(&dir).is_err() { + break; + } + } + } + fn stat_of(&self, name: &str, contents: &str) -> ConfigFileStat { let modified = std::fs::metadata(self.dir.join(name)) .ok() @@ -138,9 +151,9 @@ impl Drop for Staging { impl ConfigStorePort for FsConfigStore { fn stat_all(&self) -> Result, String> { let mut stats = Vec::new(); - for name in CONFIG_FILES { - match std::fs::read_to_string(self.dir.join(name)) { - Ok(contents) => stats.push(self.stat_of(name, &contents)), + for name in config_files(&self.dir)? { + match std::fs::read_to_string(self.dir.join(&name)) { + Ok(contents) => stats.push(self.stat_of(&name, &contents)), Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => return Err(format!("read '{}': {}", name, e)), } @@ -188,7 +201,12 @@ impl ConfigStorePort for FsConfigStore { )]) })?; for (name, contents) in &files { - std::fs::write(stage.path.join(name), contents) + let path = stage.path.join(name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| ConfigErrors::new(vec![format!("stage '{}': {}", name, e)]))?; + } + std::fs::write(&path, contents) .map_err(|e| ConfigErrors::new(vec![format!("stage '{}': {}", name, e)]))?; } @@ -215,7 +233,16 @@ impl ConfigStorePort for FsConfigStore { if std::fs::read_to_string(&target).is_ok_and(|current| current == *contents) { continue; } - let tmp = self.dir.join(format!(".{}.tmp", name)); + // The temp file sits next to its target (a rename is only atomic + // within one filesystem) and is hidden, so the loader skips it. + let parent = target.parent().unwrap_or(&self.dir).to_path_buf(); + std::fs::create_dir_all(&parent) + .map_err(|e| format!("create '{}': {}", parent.display(), e))?; + let file_name = target + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + let tmp = parent.join(format!(".{}.tmp", file_name)); std::fs::write(&tmp, contents) .map_err(|e| format!("write '{}': {}", tmp.display(), e))?; staged.push((tmp, target)); @@ -233,12 +260,12 @@ impl ConfigStorePort for FsConfigStore { // Whatever `files` does not contain is gone — either explicitly // deleted or pruned away by a whole-directory push. - for name in CONFIG_FILES { - if files.contains_key(name) { + for name in config_files(&self.dir)? { + if files.contains_key(&name) { continue; } - match std::fs::remove_file(self.dir.join(name)) { - Ok(()) => {} + match std::fs::remove_file(self.dir.join(&name)) { + Ok(()) => self.remove_empty_parents(&name), Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => return Err(format!("remove '{}': {}", name, e)), } @@ -281,11 +308,11 @@ mod tests { let store = FsConfigStore::new(dir.path()); let before = std::fs::read_to_string(dir.path().join("config.yaml")).expect("read"); - // A source pointing at a project nobody declared: valid YAML, invalid + // A connector pointing at a project nobody declared: valid YAML, invalid // configuration — the case that must never reach the disk. let change = writing(&[( - "sources.yaml", - "src-a:\n name: \"A\"\n project_id: \"prj-ghost\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", + "connectors.yaml", + "connectors:\n - name: src-a\n description: \"A\"\n project: \"prj-ghost\"\n path: \"x.py\"\n", )]); let errors = store.load(&change).err().expect("must be rejected"); @@ -294,7 +321,7 @@ mod tests { "errors: {:?}", errors.errors ); - assert!(!dir.path().join("sources.yaml").exists()); + assert!(!dir.path().join("connectors.yaml").exists()); assert_eq!( std::fs::read_to_string(dir.path().join("config.yaml")).expect("read"), before @@ -306,7 +333,7 @@ mod tests { let dir = dir_with_minimal_config(); let store = FsConfigStore::new(dir.path()); - let _ = store.load(&writing(&[("sources.yaml", "not: [valid\n")])); + let _ = store.load(&writing(&[("connectors.yaml", "not: [valid\n")])); let _ = store.load(&writing(&[("enrichers.yaml", "")])); let leftovers: Vec<_> = std::fs::read_dir(dir.path()) @@ -325,7 +352,7 @@ mod tests { let change = writing(&[( "credentials.yaml", - "cred-a:\n name: \"A\"\n type: \"token\"\n env_prefix: \"A\"\n", + "credentials:\n - name: cred-a\n description: \"A\"\n type: \"token\"\n env_prefix: \"A\"\n", )]); store.load(&change).expect("valid"); store.commit(&change).expect("commits"); @@ -361,7 +388,7 @@ mod tests { std::thread::sleep(std::time::Duration::from_millis(50)); let change = writing(&[( "credentials.yaml", - "cred-a:\n name: \"A\"\n type: \"token\"\n env_prefix: \"A\"\n", + "credentials:\n - name: cred-a\n description: \"A\"\n type: \"token\"\n env_prefix: \"A\"\n", )]); store.commit(&change).expect("commits"); @@ -392,7 +419,7 @@ mod tests { let dir = dir_with_minimal_config(); std::fs::write( dir.path().join("credentials.yaml"), - "cred-a:\n name: \"A\"\n type: \"token\"\n env_prefix: \"A\"\n", + "credentials:\n - name: cred-a\n description: \"A\"\n type: \"token\"\n env_prefix: \"A\"\n", ) .expect("write"); let store = FsConfigStore::new(dir.path()); @@ -443,12 +470,12 @@ mod tests { let dir = dir_with_minimal_config(); std::fs::write( dir.path().join("projects.yaml"), - "prj-a:\n name: \"A\"\n git_url: \"https://example.invalid/a.git\"\n", + "projects:\n - name: prj-a\n git_url: \"https://example.invalid/a.git\"\n", ) .expect("write"); std::fs::write( - dir.path().join("sources.yaml"), - "src-a:\n name: \"A\"\n project_id: \"prj-a\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", + dir.path().join("connectors.yaml"), + "connectors:\n - name: src-a\n project: \"prj-a\"\n path: \"x.py\"\n", ) .expect("write"); let store = FsConfigStore::new(dir.path()); @@ -473,14 +500,14 @@ mod tests { let errors = store .load(&writing(&[( - "sources.yaml", - "src-a:\n ttl_seconds: \"not a number\"\n", + "connectors.yaml", + "connectors:\n - name: src-a\n timeout_seconds: \"not a number\"\n", )])) .err() .expect("must be rejected"); assert!( - errors.errors[0].starts_with("sources.yaml:"), + errors.errors[0].starts_with("connectors.yaml:"), "a parse error has to say which of eight files it is about: {:?}", errors.errors ); @@ -525,4 +552,39 @@ mod tests { errors.errors ); } + + #[test] + fn files_in_folders_are_written_and_pruned_folders_go_away() { + let dir = dir_with_minimal_config(); + let store = FsConfigStore::new(dir.path()); + let change = writing(&[( + "credentials/team-a/cred-a.yaml", + "credentials:\n - name: cred-a\n type: token\n env_prefix: A\n", + )]); + let cfg = store.load(&change).expect("valid"); + assert!(cfg.credentials.contains_key("cred-a")); + store.commit(&change).expect("commit"); + assert!(dir.path().join("credentials/team-a/cred-a.yaml").exists()); + let names: Vec = store + .stat_all() + .unwrap() + .into_iter() + .map(|s| s.name) + .collect(); + assert!( + names.contains(&"credentials/team-a/cred-a.yaml".to_string()), + "{names:?}" + ); + + // A whole-directory push without it: the file goes, and so do the + // folders it leaves empty. + let config = std::fs::read_to_string(dir.path().join("config.yaml")).unwrap(); + let prune = ConfigChange { + write: [("config.yaml".to_string(), config)].into_iter().collect(), + prune: true, + ..ConfigChange::default() + }; + store.commit(&prune).expect("commit"); + assert!(!dir.path().join("credentials").exists()); + } } diff --git a/src/adapters/out/git/cli.rs b/src/adapters/out/git/cli.rs index 846cc07..41bb4f9 100644 --- a/src/adapters/out/git/cli.rs +++ b/src/adapters/out/git/cli.rs @@ -90,6 +90,22 @@ impl GitPort for CliGit { Ok(()) }) } + + fn revision(&self, dir: &Path) -> crate::ports::git::RevisionFuture<'_> { + let dir = dir.to_path_buf(); + Box::pin(async move { + let mut cmd = Command::new("git"); + cmd.args(["-C", dir.to_str()?, "rev-parse", "HEAD"]); + cmd.env("GIT_TERMINAL_PROMPT", "0"); + cmd.kill_on_drop(true); + let output = cmd.output().await.ok()?; + if !output.status.success() { + return None; + } + let sha = String::from_utf8_lossy(&output.stdout).trim().to_string(); + (!sha.is_empty()).then_some(sha) + }) + } } fn path_str(path: &Path) -> Result<&str, GitError> { diff --git a/src/adapters/out/peers/http.rs b/src/adapters/out/peers/http.rs index 4f957b8..6560ea2 100644 --- a/src/adapters/out/peers/http.rs +++ b/src/adapters/out/peers/http.rs @@ -2,8 +2,8 @@ use std::time::Duration; use serde::Deserialize; -use crate::domain::topology::Topology; -use crate::ports::peer::{PeerFuture, PeerPort, PeerRequest}; +use crate::domain::topology::{FederationPush, Topology}; +use crate::ports::peer::{PeerFuture, PeerPort, PeerRequest, PushError, PushFuture}; // PeerPort → the peer's own HTTP API, with the key the remote sources use. // @@ -53,6 +53,15 @@ impl PeerPort for HttpPeers { }; Box::pin(describe(client, request)) } + + fn push(&self, request: PeerRequest, body: FederationPush) -> PushFuture { + let client = if request.insecure_tls { + self.insecure.clone() + } else { + self.verified.clone() + }; + Box::pin(push(client, request, body)) + } } #[derive(Deserialize)] @@ -124,6 +133,7 @@ async fn ask(client: &reqwest::Client, request: &PeerRequest) -> Result Result Result<(), PushError> { + let mut call = client + .put(format!("{}/api/v1/topology/federation", request.url)) + .timeout(Duration::from_secs(request.timeout_seconds)) + .json(&body); + if let Some(key) = &request.api_key { + call = call.header("x-api-key", key); + } + let response = call + .send() + .await + .map_err(|e| PushError::Failed(unreachable_message(&e)))?; + match response.status().as_u16() { + 200..=299 => Ok(()), + // Older than the route: a 404, or a 405 where only GET /topology + // answers under that path's prefix. + 404 | 405 => Err(PushError::Unsupported), + 401 => Err(PushError::Failed("refused the key (401)".to_string())), + other => Err(PushError::Failed(format!("answered HTTP {other}"))), + } +} + // reqwest's error text for a timeout or a refused connection is long and // nested; the map needs the gist. fn unreachable_message(error: &reqwest::Error) -> String { diff --git a/src/application/config.rs b/src/application/config.rs index 2918beb..ccfb752 100644 --- a/src/application/config.rs +++ b/src/application/config.rs @@ -66,6 +66,7 @@ pub struct ReloadReport { // therefore need a restart to take effect pub restart_required: Vec, pub sources: SectionDelta, + pub connectors: SectionDelta, pub views: SectionDelta, pub enrichers: SectionDelta, pub endpoints: SectionDelta, @@ -158,6 +159,7 @@ pub fn apply(state: &AppState, cfg: &AppConfig) -> ReloadReport { fn runtime_config(cfg: &AppConfig) -> RuntimeConfig { RuntimeConfig { sources: cfg.sources.clone(), + connectors: cfg.connectors.clone(), credentials: cfg.credentials.clone(), views: cfg.views.clone(), enrichers: cfg.enrichers.clone(), @@ -173,6 +175,12 @@ fn runtime_config(cfg: &AppConfig) -> RuntimeConfig { refresh_max_concurrent: cfg.server.refresh_max_concurrent, ldap: cfg.server.ui.ldap.clone(), ui_home: cfg.server.ui.home().to_string(), + url: cfg + .server + .url + .as_deref() + .map(|url| crate::domain::topology::without_userinfo(url.trim_end_matches('/'))), + name: cfg.server.name.clone(), } } @@ -186,6 +194,7 @@ fn diff( ) -> ReloadReport { let mut report = ReloadReport { sources: SectionDelta::between(&previous.sources, &next.sources), + connectors: SectionDelta::between(&previous.connectors, &next.connectors), views: SectionDelta::between(&previous.views, &next.views), enrichers: SectionDelta::between(&previous.enrichers, &next.enrichers), endpoints: SectionDelta::between(&previous.endpoints, &next.endpoints), @@ -197,6 +206,7 @@ fn diff( let mut applied = Vec::new(); for (name, delta) in [ ("sources", &report.sources), + ("connectors", &report.connectors), ("views", &report.views), ("enrichers", &report.enrichers), ("endpoints", &report.endpoints), @@ -225,6 +235,12 @@ fn diff( if previous.shutdown_grace_seconds != next.shutdown_grace_seconds { applied.push("server.shutdown_grace_seconds".to_string()); } + if previous.url != next.url { + applied.push("server.url".to_string()); + } + if previous.name != next.name { + applied.push("server.name".to_string()); + } if previous.refresh_timeout_seconds != next.refresh_timeout_seconds { applied.push("server.refresh_timeout_seconds".to_string()); } diff --git a/src/application/enrich.rs b/src/application/enrich.rs index 5787c18..40f06c5 100644 --- a/src/application/enrich.rs +++ b/src/application/enrich.rs @@ -505,7 +505,7 @@ mod tests { } fn script_enricher() -> Enricher { - serde_saphyr::from_str("name: spy\ntarget_id: src-a\nscript_path: /bin/true\n") + serde_saphyr::from_str("name: spy\ntarget: src-a\nscript_path: /bin/true\n") .expect("enricher fixture") } @@ -672,10 +672,8 @@ mod tests { } fn declarative_enricher() -> Enricher { - serde_saphyr::from_str( - "name: d\ntarget_id: src-a\nsource_id: src-b\nfields: [\"infinibox\"]\n", - ) - .expect("enricher fixture") + serde_saphyr::from_str("name: d\ntarget: src-a\nsource: src-b\nfields: [\"infinibox\"]\n") + .expect("enricher fixture") } #[tokio::test] @@ -960,8 +958,7 @@ mod tests { } fn enricher_without_fields() -> Enricher { - serde_saphyr::from_str("name: d\ntarget_id: src-a\nsource_id: src-b\n") - .expect("enricher fixture") + serde_saphyr::from_str("name: d\ntarget: src-a\nsource: src-b\n").expect("enricher fixture") } // No `fields` means every var the source declares -- the way being in a @@ -1063,7 +1060,7 @@ mod tests { ); let enricher: Enricher = - serde_saphyr::from_str("name: d\ntarget_id: src-a\nsource_id: src-b\nfields: []\n") + serde_saphyr::from_str("name: d\ntarget: src-a\nsource: src-b\nfields: []\n") .expect("enricher fixture"); run_enricher( @@ -1083,11 +1080,8 @@ mod tests { } fn enricher_yaml(extra: &str) -> Enricher { - serde_saphyr::from_str(&format!( - "name: d\ntarget_id: src-a\nsource_id: src-b\n{}", - extra - )) - .expect("enricher fixture") + serde_saphyr::from_str(&format!("name: d\ntarget: src-a\nsource: src-b\n{}", extra)) + .expect("enricher fixture") } // Two sources of vars, one target, and the real shape of the problem: the @@ -1287,7 +1281,7 @@ mod tests { // Config validation refuses this at startup, but the domain type can // exist: the use case must answer with words, not a panic let enricher: Enricher = - serde_saphyr::from_str("name: bare\ntarget_id: src-a\n").expect("fixture"); + serde_saphyr::from_str("name: bare\ntarget: src-a\n").expect("fixture"); let outcome = run_enricher( &cache, &SpyEnricher::default(), @@ -1321,7 +1315,7 @@ mod tests { }; // project_id exercises the checkout resolution branch on the way in let enricher: Enricher = serde_saphyr::from_str( - "name: hung\ntarget_id: src-a\nscript_path: /bin/true\nproject_id: prj-x\ntimeout_seconds: 5\n", + "name: hung\ntarget: src-a\nscript_path: /bin/true\nproject: prj-x\ntimeout_seconds: 5\n", ) .expect("fixture"); diff --git a/src/application/host_search.rs b/src/application/host_search.rs index daf24cd..172c0c9 100644 --- a/src/application/host_search.rs +++ b/src/application/host_search.rs @@ -234,7 +234,7 @@ mod tests { fn endpoint(sources: &[&str], limit: Option<&str>) -> OutputEndpoint { let mut yaml = format!( - "name: e\nsource_ids: [{}]\noutput: ansible\n", + "name: e\nsources: [{}]\noutput: ansible\n", sources.join(", ") ); if let Some(l) = limit { diff --git a/src/application/projects.rs b/src/application/projects.rs index 7a64c98..3b181ea 100644 --- a/src/application/projects.rs +++ b/src/application/projects.rs @@ -255,7 +255,7 @@ mod tests { let git = StubGit { fail_with: None }; let project: GitProject = serde_saphyr::from_str( - "name: Test\ngit_url: \"https://example.com/repo.git\"\ncredential_id: \"cred-ghost\"\n", + "name: Test\ngit_url: \"https://example.com/repo.git\"\ncredential: \"cred-ghost\"\n", ) .expect("project fixture"); diff --git a/src/application/topology.rs b/src/application/topology.rs index fbe511a..6e5744a 100644 --- a/src/application/topology.rs +++ b/src/application/topology.rs @@ -1,11 +1,14 @@ use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; use crate::domain::source::Source; use crate::domain::sync_health::SyncHealthRegistry; -use crate::domain::topology::{Peer, PeerSource, Topology, peers_of}; +use crate::domain::topology::{ + Federation, FederationPush, InstanceSummary, Peer, PeerSource, Topology, peers_of, +}; use crate::ports::cache::CachePort; -use crate::ports::peer::{PeerPort, PeerRequest}; +use crate::ports::peer::{PeerPort, PeerRequest, PushError}; use crate::ports::secrets::SecretsPort; // The use case "describe the federation from here". @@ -32,12 +35,15 @@ fn peer_timeout(depth: u8) -> u64 { PEER_TIMEOUT_SECONDS * (u64::from(depth) + 1) } +#[derive(Clone)] pub struct Inputs<'a> { pub sources: &'a HashMap, pub cache: &'a dyn CachePort, pub health: &'a SyncHealthRegistry, pub secrets: &'a dyn SecretsPort, pub peers: Arc, + // This instance's `server.name`, for its own answer. + pub name: Option<&'a str>, } pub async fn describe(inputs: Inputs<'_>, visible: impl Fn(&str) -> bool, depth: u8) -> Topology { @@ -64,6 +70,7 @@ pub async fn describe(inputs: Inputs<'_>, visible: impl Fn(&str) -> bool, depth: groups: entry.as_ref().map(|e| e.dataset.groups.len()), age_seconds: entry.as_ref().map(|e| e.age_seconds()), consecutive_failures: health.consecutive_failures, + last_pull_age_seconds: health.last_success_age_seconds, last_error: health.last_error, } }) @@ -82,8 +89,7 @@ pub async fn describe(inputs: Inputs<'_>, visible: impl Fn(&str) -> bool, depth: // The key and TLS setting the sources reading this peer already use: // whatever lets them pull data lets the map ask who it is. Resolved // below, once every peer is listed. - let (_, first) = members[0]; - to_ask.push((index, url.clone(), first)); + to_ask.push((index, url.clone(), key_source(&members))); peers.push(Peer { url, sources, @@ -94,39 +100,14 @@ pub async fn describe(inputs: Inputs<'_>, visible: impl Fn(&str) -> bool, depth: // One after another: resolutions come from the secrets cache (env vars // are instant), and it is the peer requests below that run concurrently. - let mut keys = Vec::with_capacity(to_ask.len()); - for (_, _, first) in &to_ask { - keys.push( - crate::application::credentials::resolve_credentials( - inputs.secrets, - &first.credential_ids, - ) - .await, - ); - } - - for ((index, url, first), resolved) in to_ask.into_iter().zip(keys) { - let api_key = match resolved { - Ok(credentials) => credentials - .get("token") - .or_else(|| credentials.get("TOKEN")) - .or_else(|| credentials.get("api_key")) - .cloned(), - Err(e) => { - peers[index].error = Some(format!("not asked: {}", e.message)); + for (index, url, first) in to_ask { + let request = match peer_request(inputs.secrets, url, first, depth - 1).await { + Ok(request) => request, + Err(reason) => { + peers[index].error = Some(reason); continue; } }; - let request = PeerRequest { - url, - api_key, - insecure_tls: first - .config - .get("insecure_tls") - .is_some_and(|v| v == "true"), - timeout_seconds: peer_timeout(depth - 1), - depth: depth - 1, - }; let port = Arc::clone(&inputs.peers); asks.spawn(async move { (index, port.describe(request).await) }); asked.push(index); @@ -156,12 +137,180 @@ pub async fn describe(inputs: Inputs<'_>, visible: impl Fn(&str) -> bool, depth: } Topology { + name: inputs.name.map(str::to_string), version: env!("CARGO_PKG_VERSION").to_string(), sources: Some(visible_count), peers, } } +// Which of the sources reading a peer lends the map its key and TLS setting: +// the first by id (peers_of sorts them). One choice for asking a peer and for +// pushing to it — when two sources with different credentials read one peer, +// a push picked by HashMap order could use another key than the map was +// built with (a broken one, even), and flip between them across reloads, +// leaving the peer two maps under two key names. +fn key_source<'s>(members: &[(&str, &'s Source)]) -> &'s Source { + members[0].1 +} + +// How to reach a peer: the key and TLS setting of the first source reading +// from it, since whatever lets them pull data lets the map ask who it is (and +// tell it where it sits). `depth` is what the peer may spend asking further. +async fn peer_request( + secrets: &dyn SecretsPort, + url: String, + source: &Source, + depth: u8, +) -> Result { + let credentials = + crate::application::credentials::resolve_credentials(secrets, &source.credential_ids) + .await + .map_err(|e| format!("not asked: {}", e.message))?; + let api_key = credentials + .get("token") + .or_else(|| credentials.get("TOKEN")) + .or_else(|| credentials.get("api_key")) + .cloned(); + Ok(PeerRequest { + url, + api_key, + insecure_tls: source + .config + .get("insecure_tls") + .is_some_and(|v| v == "true"), + timeout_seconds: peer_timeout(depth), + depth, + }) +} + +// How deep the pushed map goes: the same default the Topology page asks for. +const PUSH_DEPTH: u8 = 2; + +// The use case "tell every peer where it sits": map the federation from here +// (all sources), then send it to each peer that answered. One that did not is +// already on the map as unreachable; there is nobody to tell. What is sent is +// the map as a peer's admins may see it (Topology::for_push), not ours: our +// peers' own maps were asked with our keys, and our errors name our secrets. +// Returns each push's outcome, by URL, for the caller to log. +pub async fn push_federation( + inputs: Inputs<'_>, + sender_url: Option<&str>, + endpoints: usize, +) -> Vec<(String, Result<(), PushError>)> { + let sender = summarize(inputs.sources, inputs.cache, endpoints); + let topology = describe(inputs.clone(), |_| true, PUSH_DEPTH).await; + let grouped = peers_of(inputs.sources, |_| true); + let answered: Vec = topology + .peers + .iter() + .filter(|p| p.instance.is_some()) + .map(|p| p.url.clone()) + .collect(); + let pushed = topology.for_push(); + let mut pushes = tokio::task::JoinSet::new(); + let mut outcomes = Vec::new(); + + for url in answered { + let Some(members) = grouped.get(&url) else { + continue; + }; + match peer_request(inputs.secrets, url.clone(), key_source(members), 0).await { + Ok(request) => { + let body = FederationPush { + url: url.clone(), + sender_url: sender_url.map(str::to_string), + sender: sender.clone(), + topology: pushed.clone(), + }; + let port = Arc::clone(&inputs.peers); + pushes.spawn(async move { (url, port.push(request, body).await) }); + } + Err(reason) => outcomes.push((url, Err(PushError::Failed(reason)))), + } + } + while let Some(joined) = pushes.join_next().await { + if let Ok(outcome) = joined { + outcomes.push(outcome); + } + } + outcomes.sort_by(|a, b| a.0.cmp(&b.0)); + outcomes +} + +// This instance's own card, as an edge draws it: the facts the instance's own +// Topology page shows for itself, from its configuration and cache. +fn summarize( + sources: &HashMap, + cache: &dyn CachePort, + endpoints: usize, +) -> InstanceSummary { + let mut summary = InstanceSummary { + sources: sources.keys().cloned().collect(), + endpoints, + ..Default::default() + }; + summary.sources.sort(); + for id in sources.keys() { + if let Some(entry) = cache.get(id) { + summary.hosts += entry.dataset.hostvars.len(); + summary.groups += entry.dataset.groups.len(); + let age = entry.age_seconds(); + summary.oldest_age_seconds = + Some(summary.oldest_age_seconds.map_or(age, |a| a.max(age))); + } + } + summary +} + +// A pushed map is dropped once its sender has been silent this long: ten +// missed pushes, so a sender that was removed (or reconfigured to read from +// elsewhere) does not leave its picture here forever. +const FEDERATION_EXPIRES: Duration = Duration::from_secs(600); + +// The maps pushed here, one per sender (by the key it authenticated with: two +// instances reading from this one each keep their own). In memory only: after +// a restart the next push, within a minute, brings it back. +#[derive(Default)] +pub struct FederationStore { + pushed: RwLock>, +} + +impl FederationStore { + pub fn new() -> Self { + Self::default() + } + + pub fn record(&self, pushed_by: String, push: FederationPush) { + let mut pushed = self.pushed.write().expect("federation store"); + pushed.retain(|_, (_, at)| at.elapsed() < FEDERATION_EXPIRES); + pushed.insert(pushed_by, (push, Instant::now())); + } + + // Newest first: the sender pushing now is the likeliest to be current. + pub fn list(&self) -> Vec { + let pushed = self.pushed.read().expect("federation store"); + let mut list: Vec = pushed + .iter() + .filter(|(_, (_, at))| at.elapsed() < FEDERATION_EXPIRES) + .map(|(by, (push, at))| Federation { + pushed_by: by.clone(), + url: push.url.clone(), + sender_url: push.sender_url.clone(), + sender: push.sender.clone(), + topology: push.topology.clone(), + age_seconds: at.elapsed().as_secs(), + }) + .collect(); + list.sort_by(|a, b| { + a.age_seconds + .cmp(&b.age_seconds) + .then(a.pushed_by.cmp(&b.pushed_by)) + }); + list + } +} + #[cfg(test)] mod tests { use super::*; @@ -175,10 +324,15 @@ mod tests { } use crate::adapters::out::cache::memory::MemoryCache; use crate::adapters::out::secrets::mock::MockSecrets; - use crate::ports::peer::PeerFuture; + use crate::ports::peer::{PeerFuture, PushFuture}; + use std::sync::Mutex; - // dc1 answers as an edge with no peers of its own; dc2 is down. - struct Peers; + // dc1 answers as an edge with no peers of its own; dc2 is down. Pushes + // are recorded, by URL. + #[derive(Default)] + struct Peers { + pushed: Mutex>, + } impl PeerPort for Peers { fn describe(&self, request: PeerRequest) -> PeerFuture { @@ -188,12 +342,19 @@ mod tests { return Err("unreachable".to_string()); } Ok(Topology { + name: None, version: "0.33.0".to_string(), sources: Some(3), peers: Vec::new(), }) }) } + + fn push(&self, request: PeerRequest, body: FederationPush) -> PushFuture { + assert_eq!(request.api_key.as_deref(), Some("edge-key")); + self.pushed.lock().unwrap().push(body); + Box::pin(async { Ok(()) }) + } } fn remote(url: &str) -> Source { @@ -226,7 +387,8 @@ mod tests { cache: &cache, health: &health, secrets: &secrets, - peers: Arc::new(Peers), + peers: Arc::new(Peers::default()), + name: None, }, |_| true, 2, @@ -244,6 +406,55 @@ mod tests { assert_eq!(dc2.error.as_deref(), Some("unreachable")); } + // An instance names itself in its answer, when its server.name says. + #[tokio::test] + async fn an_instance_says_its_name() { + let sources = HashMap::new(); + let cache = MemoryCache::new(); + let health = SyncHealthRegistry::new(); + let secrets = MockSecrets::new(); + let topology = describe( + Inputs { + sources: &sources, + cache: &cache, + health: &health, + secrets: &secrets, + peers: Arc::new(Peers::default()), + name: Some("Central"), + }, + |_| true, + 0, + ) + .await; + assert_eq!(topology.name.as_deref(), Some("Central")); + } + + // When data last moved along a link: the last pull that worked, for the + // UI to animate. Absent until one has. + #[tokio::test] + async fn a_peer_source_says_when_it_last_pulled() { + let sources = sources(); + let cache = MemoryCache::new(); + let health = SyncHealthRegistry::new(); + health.record_success("src-edge-dc1"); + let secrets = MockSecrets::new().with("cred-edge", [("token", "edge-key")]); + let topology = describe( + Inputs { + sources: &sources, + cache: &cache, + health: &health, + secrets: &secrets, + peers: Arc::new(Peers::default()), + name: None, + }, + |_| true, + 0, + ) + .await; + assert_eq!(topology.peers[0].sources[0].last_pull_age_seconds, Some(0)); + assert_eq!(topology.peers[1].sources[0].last_pull_age_seconds, None); + } + // Depth 0 maps this instance's peers without asking them anything: the // hop budget ran out, and a loop of instances must end. #[tokio::test] @@ -258,7 +469,8 @@ mod tests { cache: &cache, health: &health, secrets: &secrets, - peers: Arc::new(Peers), + peers: Arc::new(Peers::default()), + name: None, }, |_| true, 0, @@ -273,4 +485,212 @@ mod tests { .contains("depth") ); } + + // Every peer that answered is told the whole map and the address it was + // reached at; the one that did not answer is not pushed to at all. + #[tokio::test] + async fn the_map_is_pushed_to_every_peer_that_answered() { + let sources = sources(); + let cache = MemoryCache::new(); + let health = SyncHealthRegistry::new(); + let secrets = MockSecrets::new().with("cred-edge", [("token", "edge-key")]); + let peers = Arc::new(Peers::default()); + let outcomes = push_federation( + Inputs { + sources: &sources, + cache: &cache, + health: &health, + secrets: &secrets, + peers: Arc::clone(&peers) as Arc, + name: None, + }, + Some("https://central.example"), + 6, + ) + .await; + + assert_eq!(outcomes.len(), 1); + assert_eq!( + outcomes[0], + ("https://dc1.example:8443".to_string(), Ok(())) + ); + let pushed = peers.pushed.lock().unwrap(); + assert_eq!(pushed[0].url, "https://dc1.example:8443"); + assert_eq!( + pushed[0].sender_url.as_deref(), + Some("https://central.example") + ); + assert_eq!(pushed[0].sender.sources, ["src-edge-dc1", "src-edge-dc2"]); + assert_eq!(pushed[0].sender.endpoints, 6); + // Nothing cached yet: no hosts, and no age to report. + assert_eq!(pushed[0].sender.hosts, 0); + assert_eq!(pushed[0].sender.oldest_age_seconds, None); + assert_eq!(pushed[0].topology.peers.len(), 2); + } + + // An edge's admins are not the central's: the map pushed to dc1 holds no + // peer's own map (asked with the central's keys) and no error naming the + // central's secrets — neither a peer's "not asked" reason nor a link's + // last sync error. + #[tokio::test] + async fn the_pushed_map_is_redacted_for_the_peer() { + struct Deep { + pushed: Mutex>, + } + impl PeerPort for Deep { + fn describe(&self, _request: PeerRequest) -> PeerFuture { + Box::pin(async { + Ok(Topology { + name: Some("dc1".to_string()), + version: "0.35.0".to_string(), + sources: Some(7), + peers: vec![Peer { + url: "https://behind-dc1.example".to_string(), + sources: Vec::new(), + instance: None, + error: Some("unreachable".to_string()), + }], + }) + }) + } + fn push(&self, _request: PeerRequest, body: FederationPush) -> PushFuture { + self.pushed.lock().unwrap().push(body); + Box::pin(async { Ok(()) }) + } + } + + let mut sources = sources(); + let mut dc3 = remote("https://dc3.example:8443"); + dc3.credential_ids = vec!["cred-missing".to_string()]; + sources.insert("src-edge-dc3".to_string(), dc3); + let cache = MemoryCache::new(); + let health = SyncHealthRegistry::new(); + health.record_failure("src-edge-dc1", "Env var 'CENTRAL_DC1_TOKEN' not set"); + let secrets = MockSecrets::new().with("cred-edge", [("token", "edge-key")]); + let peers = Arc::new(Deep { + pushed: Mutex::new(Vec::new()), + }); + let outcomes = push_federation( + Inputs { + sources: &sources, + cache: &cache, + health: &health, + secrets: &secrets, + peers: Arc::clone(&peers) as Arc, + name: None, + }, + None, + 0, + ) + .await; + assert_eq!(outcomes.len(), 2, "dc1 and dc2 answered; dc3 was not asked"); + + let pushed = peers.pushed.lock().unwrap(); + let json = serde_json::to_string(&pushed[0].topology).unwrap(); + assert!(!json.contains("CENTRAL_DC1_TOKEN"), "{json}"); + assert!(!json.contains("cred-missing"), "{json}"); + assert!(!json.contains("behind-dc1"), "{json}"); + let dc1 = &pushed[0].topology.peers[0]; + let instance = dc1.instance.as_ref().unwrap(); + assert_eq!(instance.version, "0.35.0"); + assert!(instance.peers.is_empty() && instance.sources.is_none()); + assert_eq!( + dc1.sources[0].consecutive_failures, 1, + "a failing link still shows" + ); + assert!(dc1.sources[0].last_error.is_some()); + let dc3 = &pushed[0].topology.peers[2]; + assert_eq!( + dc3.error.as_deref(), + Some("not asked: this instance cannot authenticate to it") + ); + } + + // Two sources read dc1 with different keys: the map asks with the first + // by id, and the push must use that same key, however the HashMap of + // sources happens to iterate. + #[tokio::test] + async fn the_push_uses_the_key_the_map_asked_with() { + #[derive(Default)] + struct Keys { + asked: Mutex>>, + pushed: Mutex>>, + } + impl PeerPort for Keys { + fn describe(&self, request: PeerRequest) -> PeerFuture { + self.asked.lock().unwrap().push(request.api_key); + Box::pin(async { + Ok(Topology { + name: None, + version: "0.35.0".to_string(), + sources: Some(1), + peers: Vec::new(), + }) + }) + } + fn push(&self, request: PeerRequest, _body: FederationPush) -> PushFuture { + self.pushed.lock().unwrap().push(request.api_key); + Box::pin(async { Ok(()) }) + } + } + + // A fresh HashMap per round: each gets its own iteration order. + for _ in 0..16 { + let mut first = remote("https://dc1.example:8443"); + first.credential_ids = vec!["cred-a".to_string()]; + let mut second = remote("https://dc1.example:8443/"); + second.credential_ids = vec!["cred-b".to_string()]; + let sources: HashMap = [ + ("src-dc1-b".to_string(), second), + ("src-dc1-a".to_string(), first), + ] + .into_iter() + .collect(); + let cache = MemoryCache::new(); + let health = SyncHealthRegistry::new(); + let secrets = MockSecrets::new() + .with("cred-a", [("token", "key-a")]) + .with("cred-b", [("token", "key-b")]); + let peers = Arc::new(Keys::default()); + push_federation( + Inputs { + sources: &sources, + cache: &cache, + health: &health, + secrets: &secrets, + peers: Arc::clone(&peers) as Arc, + name: None, + }, + None, + 0, + ) + .await; + let key_a = [Some("key-a".to_string())]; + assert_eq!(*peers.asked.lock().unwrap(), key_a); + assert_eq!(*peers.pushed.lock().unwrap(), key_a); + } + } + + #[test] + fn the_store_keeps_one_map_per_sender() { + let store = FederationStore::new(); + let push = |url: &str| FederationPush { + url: url.to_string(), + sender_url: None, + sender: InstanceSummary::default(), + topology: Topology { + name: None, + version: "0.35.0".to_string(), + sources: Some(1), + peers: Vec::new(), + }, + }; + store.record("key-central".to_string(), push("https://old")); + store.record("key-central".to_string(), push("https://new")); + store.record("key-other".to_string(), push("https://x")); + let list = store.list(); + assert_eq!(list.len(), 2); + let central = list.iter().find(|f| f.pushed_by == "key-central").unwrap(); + assert_eq!(central.url, "https://new"); + } } diff --git a/src/config.rs b/src/config.rs index b5a82f2..31de6f6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,14 +4,19 @@ use std::fs; use std::path::Path; use crate::domain::api_key::{ApiKeyDef, ApiKeyRole}; +use crate::domain::connector::Connector; use crate::domain::credential::Credential; use crate::domain::endpoint::{BUILTIN_SETTINGS, OutputEndpoint, renamed_setting}; use crate::domain::enricher::Enricher; use crate::domain::host_filter::HostFilter; use crate::domain::project::GitProject; -use crate::domain::source::Source; +use crate::domain::source::{Source, SourceDef}; use crate::domain::view::View; +// The one-time conversion of a 0.34 directory (id-keyed maps in eight fixed +// files) into item files. Lives beside the loader it feeds. +pub mod migrate; + pub struct AppConfig { pub server: ServerConfig, pub cache: CacheConfig, @@ -19,7 +24,11 @@ pub struct AppConfig { pub secrets_config: SecretsConfig, pub config_api: ConfigApiConfig, pub credentials: HashMap, + // Resolved: each source with its connector's settings folded in (see + // SourceDef::resolve). The connectors themselves are kept too, for the + // /connectors listing and for what a reload reports. pub sources: HashMap, + pub connectors: HashMap, pub views: HashMap, pub enrichers: HashMap, pub projects: HashMap, @@ -105,6 +114,20 @@ pub struct ServerConfig { // PUT is one body) or the largest host-vars document a consumer writes. #[serde(default = "default_max_body_bytes")] pub max_body_bytes: usize, + + // The address people and other instances reach this one at + // (https://unified-api-central.example.com). The process only knows the + // port it listens on; the real address is the ingress's. Optional, and + // only used to tell the instances this one reads from where it is, so + // their Topology page can link back to it. + #[serde(default)] + pub url: Option, + + // What this instance is called where it is shown among others (the + // Topology page): "Central", "CDI DC01". Optional; without it the UI + // derives a name from the instance's address. + #[serde(default)] + pub name: Option, } fn default_refresh_timeout_seconds() -> u64 { @@ -220,27 +243,119 @@ pub struct ConfigApiConfig { pub enabled: bool, } -// Every file `load_config` reads, in the order --check-config reports them. -// One list, so the config API cannot drift from the loader: a file added here -// becomes readable and writable over HTTP the moment the loader knows it. -pub const CONFIG_FILES: [&str; 8] = [ - "config.yaml", - "credentials.yaml", - "sources.yaml", - "views.yaml", - "enrichers.yaml", - "projects.yaml", - "endpoints.yaml", - "api_keys.yaml", -]; +// The configuration directory is laid out the way AWX config as code is: +// config.yaml at the root holds the server settings, and every other *.yaml / +// *.yml file, at any depth, holds ITEMS under a typed top-level key: +// +// # sources/acme/src-d42-acme.yaml +// sources: +// - name: src-d42-acme +// description: "Device42 Acme Inventory" +// connector: d42 +// ... +// +// The top-level key says what an item is, never the folder: folders are for +// people (by team, by estate) and may nest as deep as they like. One item per +// file is the convention; a file may hold several, of several types. `name` +// is the item's identity: what other items reference, what URLs and cache +// keys use. `description` is the label people read. +// +// Until 0.35 the directory was eight fixed files of id-keyed maps; a +// directory still in that shape is converted once, at startup (see +// config::migrate). // The one file that is not optional: without server config there is nothing -// to bind. The config API refuses to delete it for the same reason the loader -// refuses to start without it. +// to bind. Always at the root, under exactly this name. The config API +// refuses to delete it for the same reason the loader refuses to start +// without it. pub const REQUIRED_CONFIG_FILE: &str = "config.yaml"; -pub fn is_config_file(name: &str) -> bool { - CONFIG_FILES.contains(&name) +// The top-level keys an item file may use. +pub const SECTIONS: [&str; 8] = [ + "sources", + "views", + "endpoints", + "enrichers", + "connectors", + "projects", + "credentials", + "api_keys", +]; + +// Whether a path, relative to the configuration directory, can be a +// configuration file: segments of letters, digits, `-`, `_` and `.`, none +// hidden, ending in .yaml or .yml. The same rule the loader applies to what +// it finds and the config API applies to what it is asked to write, so a file +// the API cannot name cannot be part of the configuration either. +pub fn is_config_path(path: &str) -> bool { + let segments: Vec<&str> = path.split('/').collect(); + path.len() <= 512 + && segments.len() <= 16 + && segments.iter().all(|s| is_safe_id(s)) + && (path.ends_with(".yaml") || path.ends_with(".yml")) +} + +// Every configuration file under `dir`, as relative paths with `/`, sorted: +// the same directory always loads in the same order. Hidden entries are +// skipped (the config API's staging directories, the migration's backup, +// editor swap files, .git). +// +// Symlinks ARE followed: a Kubernetes ConfigMap mount is nothing but links +// into a hidden `..data` folder, and skipping them would load an empty +// configuration. The directory is the admins' to write either way; the depth +// cap is what keeps a link loop from walking forever. +pub fn config_files(dir: &Path) -> Result, String> { + let mut out = Vec::new(); + walk(dir, "", 0, &mut out)?; + out.sort(); + Ok(out) +} + +const MAX_CONFIG_DEPTH: usize = 16; + +fn walk(dir: &Path, prefix: &str, depth: usize, out: &mut Vec) -> Result<(), String> { + if depth > MAX_CONFIG_DEPTH { + return Err(format!( + "{}: folders nested more than {MAX_CONFIG_DEPTH} deep (a symlink loop?)", + dir.display() + )); + } + let entries = match fs::read_dir(dir) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound && prefix.is_empty() => { + return Ok(()); + } + Err(e) => return Err(format!("{}: {e}", dir.display())), + }; + for entry in entries.flatten() { + let name = entry.file_name().to_string_lossy().to_string(); + if name.starts_with('.') { + continue; + } + let rel = format!("{prefix}{name}"); + // metadata, not file_type: it follows a link to what it points at. + let Ok(meta) = fs::metadata(entry.path()) else { + continue; + }; + if meta.is_dir() { + walk(&entry.path(), &format!("{rel}/"), depth + 1, out)?; + } else if meta.is_file() && (name.ends_with(".yaml") || name.ends_with(".yml")) { + out.push(rel); + } + } + Ok(()) +} + +// An identity that is also safe as a path segment and in a URL: letters, +// digits, `-`, `_` and `.`, not starting with a dot. Item names are held to +// it because they appear in both (`/api/v1/sources/{id}`, the migration's +// `
/.yaml`), and configuration paths are made of it. +pub fn is_safe_id(id: &str) -> bool { + !id.is_empty() + && !id.starts_with('.') + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) } // The settings a running process cannot adopt without being restarted, pulled @@ -432,6 +547,14 @@ impl AppConfig { pub fn validate_errors(&self) -> Vec { let mut errors: Vec = Vec::new(); + if let Some(url) = &self.server.url + && !(url.starts_with("https://") || url.starts_with("http://")) + { + errors.push(format!( + "server.url '{url}' must be an http:// or https:// address" + )); + } + // Enrichers must reference existing sources for (id, enricher) in &self.enrichers { if self.views.contains_key(&enricher.target_id) { @@ -459,7 +582,7 @@ impl AppConfig { } if enricher.source_id.is_none() && enricher.script_path.is_none() { errors.push(format!( - "Enricher '{}' needs either source_id (declarative) or script_path (script)", + "Enricher '{}' needs either source (declarative) or script_path (script)", id )); } @@ -483,7 +606,7 @@ impl AppConfig { if endpoint.output.is_some() { if endpoint.project_id.is_some() { errors.push(format!( - "Endpoint '{}' sets project_id, which a builtin output ignores", + "Endpoint '{}' sets project, which a builtin output ignores", id )); } @@ -572,7 +695,7 @@ impl AppConfig { { errors.push(format!( "Endpoint '{}' limits by hosts from '{}', which is not one of its \ - source_ids — add it to source_ids", + sources — add it to sources", id, from )); } @@ -651,7 +774,7 @@ impl AppConfig { } } - // Sources with credential_ids must reference existing credentials + // Sources' credentials must exist for (id, source) in &self.sources { for cred_id in &source.credential_ids { if !self.credentials.contains_key(cred_id) { @@ -663,13 +786,31 @@ impl AppConfig { } } - // Sources must reference existing projects — the checkout of that - // project is where a relative script_path resolves first. + // Projects must exist — the checkout of that project is where a + // relative path resolves first. A connector names its own; of + // the sources only a static_inventory one names a project (the others + // took theirs from a connector already checked here, and reporting it + // again per source would bury the one line that needs fixing). + let mut connector_ids: Vec<&String> = self.connectors.keys().collect(); + connector_ids.sort(); + for id in connector_ids { + if let Some(project) = &self.connectors[id].project + && !self.projects.contains_key(project) + { + errors.push(format!( + "Connector '{}' references unknown project '{}'", + id, project + )); + } + } for (id, source) in &self.sources { - if !self.projects.contains_key(&source.project_id) { + if source.connector_type == crate::domain::source::ConnectorType::StaticInventory + && let Some(project) = &source.project_id + && !self.projects.contains_key(project) + { errors.push(format!( "Source '{}' references unknown project '{}'", - id, source.project_id + id, project )); } } @@ -714,7 +855,8 @@ impl AppConfig { && scope.hosts.is_empty() { errors.push(format!( - "Source '{}' has an empty advertise_scope — name at least one group or host, or remove the block", + "Source '{}' has an empty advertise_scope — name at least one \ + group or host, or remove the block", id )); } @@ -781,34 +923,6 @@ impl AppConfig { } } - // host_args are CLI arguments for a spawned script. The other - // connectors are not spawned with any (ssh builds its own remote - // command per host, static_inventory reads a file, remote speaks - // HTTP), so declaring them there is a config that looks active and - // is not. - for (id, source) in &self.sources { - if !source.host_args.is_empty() - && source.connector_type != crate::domain::source::ConnectorType::Script - { - errors.push(format!( - "Source '{}' sets host_args but is not a script source", - id - )); - } - } - - // Remote (federation) sources need the remote base URL - for (id, source) in &self.sources { - if source.connector_type == crate::domain::source::ConnectorType::Remote - && !source.config.contains_key("url") - { - errors.push(format!( - "Source '{}' is a remote source but has no 'url' in config", - id - )); - } - } - // Enrichers and endpoints with a project must reference an existing one for (id, enricher) in &self.enrichers { if let Some(ref project_id) = enricher.project_id @@ -1050,17 +1164,71 @@ pub fn load_config(config_dir: &str) -> Result Result { // config.yaml is mandatory — without server config we cannot start - let server_file: ServerFile = load_yaml_file(&dir.join("config.yaml")) - .map_err(|e| ConfigErrors::single(format!("config.yaml: {}", e)))?; - - // The rest are optional — if they do not exist, empty HashMap - let credentials = load_named(dir, "credentials.yaml")?; - let sources = load_named(dir, "sources.yaml")?; - let views = load_named(dir, "views.yaml")?; - let enrichers = load_named(dir, "enrichers.yaml")?; - let projects = load_named(dir, "projects.yaml")?; - let endpoints = load_named(dir, "endpoints.yaml")?; - let api_keys = load_named(dir, "api_keys.yaml")?; + let server_file: ServerFile = load_yaml_file(&dir.join(REQUIRED_CONFIG_FILE)) + .map_err(|e| ConfigErrors::single(format!("{REQUIRED_CONFIG_FILE}: {e}")))?; + + // Every other file holds items. All of them are read before anything is + // reported, so a push with three typos in three files names all three. + let mut errors = Vec::new(); + let mut entries: HashMap<&str, Vec> = HashMap::new(); + let files = config_files(dir).map_err(ConfigErrors::single)?; + for file in files.iter().filter(|f| *f != REQUIRED_CONFIG_FILE) { + if !is_config_path(file) { + errors.push(format!( + "{file}: rename it — configuration paths may only use letters, digits, \ + '-', '_' and '.', in segments that do not start with a dot" + )); + continue; + } + match fs::read_to_string(dir.join(file)) { + Ok(text) => read_items(file, &text, &mut entries, &mut errors), + Err(e) => errors.push(format!("{file}: {e}")), + } + } + let mut section = |name: &str| entries.remove(name).unwrap_or_default(); + let (cr, pr, co, so, vi, en, ep, ak) = ( + section("credentials"), + section("projects"), + section("connectors"), + section("sources"), + section("views"), + section("enrichers"), + section("endpoints"), + section("api_keys"), + ); + let credentials = take("credential", cr, &mut errors); + let projects = take("project", pr, &mut errors); + let connectors: HashMap = take("connector", co, &mut errors); + let source_defs: HashMap = take("source", so, &mut errors); + let views = take("view", vi, &mut errors); + let enrichers = take("enricher", en, &mut errors); + let endpoints = take("endpoint", ep, &mut errors); + let api_keys = take("API key", ak, &mut errors); + + // A connector's own shape first: a source cannot be resolved against a + // connector that is itself wrong, and its errors would only repeat. + for (id, connector) in &connectors { + errors.extend(connector.errors(id)); + } + if !errors.is_empty() { + // Cross-references are only worth checking once every item parsed: + // otherwise one typo'd connector also reports every source using it. + return Err(ConfigErrors::new(errors)); + } + + let mut sources = HashMap::new(); + for (id, def) in &source_defs { + match def.resolve(id, &connectors) { + Ok(source) => { + sources.insert(id.clone(), source); + } + Err(found) => errors.extend(found), + } + } + if !errors.is_empty() { + errors.sort(); + return Err(ConfigErrors::new(errors)); + } let config = AppConfig { server: server_file.server, @@ -1070,6 +1238,7 @@ pub fn load_config_detailed(dir: &Path) -> Result { config_api: server_file.config_api, credentials, sources, + connectors, views, enrichers, projects, @@ -1085,15 +1254,183 @@ pub fn load_config_detailed(dir: &Path) -> Result { } } -// An optional file, with its name in front of whatever went wrong. A YAML -// parse error carries a line and a column and no filename at all, which in a -// directory of eight files is half an answer. -fn load_named( - dir: &Path, - name: &str, -) -> Result, ConfigErrors> { - load_optional_yaml(&dir.join(name)) - .map_err(|e| ConfigErrors::single(format!("{}: {}", name, e))) +// One item as found: which file, its name, and the rest of its keys. +struct Entry { + file: String, + name: String, + body: serde_json::Map, + // The same item with numbers and booleans as their source text + // (domain::yaml::to_text_json): what a field that wants text reads. + text: serde_json::Map, +} + +// Reads one item file into `entries`, by section. YAML is read with the +// core-schema reader static inventories use (domain::yaml), so `0644` stays +// a string, and an error names a position, never the text — a configuration +// file can hold a secret, and the error goes to the log. It is read twice: +// typed, and with every scalar as its text, which is what a text field (a +// `config:` value, a script argument) is given — `1.10` stays "1.10". +fn read_items( + file: &str, + text: &str, + entries: &mut HashMap<&'static str, Vec>, + errors: &mut Vec, +) { + use serde_json::Value; + let top = match crate::domain::yaml::to_json(text) { + Ok(Value::Object(top)) => top, + // An empty file is a file with nothing in it yet, not an error. + Ok(Value::Null) => return, + Ok(_) => { + errors.push(format!( + "{file}: expected a mapping with one of: {}", + SECTIONS.join(", ") + )); + return; + } + Err(e) => { + errors.push(format!("{file}: {e}")); + return; + } + }; + // The same document, so it parses the same; an empty map only if not. + let mut text_top = match crate::domain::yaml::to_text_json(text) { + Ok(Value::Object(top)) => top, + _ => serde_json::Map::new(), + }; + for (key, value) in top { + let mut text_items = match text_top.remove(&key) { + Some(Value::Array(items)) => items.into_iter(), + _ => Vec::new().into_iter(), + }; + let Some(section) = SECTIONS.iter().find(|s| **s == key) else { + // An id with a map under it is the 0.34 layout, most likely pushed + // by a pipeline that has not converted its files yet. Saying so + // is the difference between a typo hunt and a one-line fix. + let hint = if value.is_object() { + " — this is the 0.34 layout (items keyed by id); convert it with \ + `unified-api --migrate-config `" + } else { + "" + }; + errors.push(format!( + "{file}: unknown key '{key}' (expected one of: {}){hint}", + SECTIONS.join(", ") + )); + continue; + }; + let items = match value { + Value::Array(items) => items, + Value::Null => continue, + _ => { + errors.push(format!("{file}: '{key}' must be a list of items")); + continue; + } + }; + for (i, item) in items.into_iter().enumerate() { + let mut text_body = match text_items.next() { + Some(Value::Object(body)) => body, + _ => serde_json::Map::new(), + }; + let Value::Object(mut body) = item else { + errors.push(format!("{file}: {key} item {} is not a mapping", i + 1)); + continue; + }; + body.remove("name"); + // By its text: `name: 2024` is the item "2024", as an id-keyed + // 0.34 item of that name was. + let name = match text_body.remove("name") { + Some(Value::String(name)) if !name.trim().is_empty() => name, + _ => { + errors.push(format!( + "{file}: {key} item {} needs a 'name', its identity", + i + 1 + )); + continue; + } + }; + if !is_safe_id(&name) { + errors.push(format!( + "{file}: {key} item '{name}' — a name may only use letters, digits, \ + '-', '_' and '.', and may not start with a dot (it is used in URLs)" + )); + continue; + } + entries.entry(section).or_default().push(Entry { + file: file.to_string(), + name, + body, + text: text_body, + }); + } + } +} + +// One section's items, typed. A name defined twice is an error naming both +// files: which one wins would otherwise depend on the order files are read. +// +// Two adjustments before an item is typed: +// - its `description` becomes the struct's `name` (falling back to the +// identity). Every item struct's `name` is the label people read; the +// identity is the map key. Doing it here, once, is what keeps every +// response that has always answered `name` answering the same thing. +// - a field that wants text takes a scalar's source text (see +// domain::yaml::from_json_with_text). `config:` maps are free-form +// string → string (they become environment variables and script settings), +// and the typed reader before 0.35 took `concurrency: 20` as "20" and +// `version: 1.10` as "1.10"; read as numbers they were refused ("expected a +// string") or rewritten (1.1). The same holds for `script_args: [7]`. +fn take( + label: &str, + entries: Vec, + errors: &mut Vec, +) -> HashMap { + use serde_json::Value; + let mut out = HashMap::new(); + let mut seen: HashMap = HashMap::new(); + for mut entry in entries { + if let Some(first) = seen.get(&entry.name) { + errors.push(format!( + "{}: {label} '{}' is already defined in {first}", + entry.file, entry.name + )); + continue; + } + seen.insert(entry.name.clone(), entry.file.clone()); + + let text_description = entry.text.remove("description"); + let display = match entry.body.remove("description") { + Some(Value::String(text)) if !text.trim().is_empty() => text, + // `description: 2024`: its text, as any text field. + Some(Value::Number(_) | Value::Bool(_)) => match text_description { + Some(Value::String(text)) => text, + _ => entry.name.clone(), + }, + Some(Value::Null) | None => entry.name.clone(), + Some(Value::String(_)) => entry.name.clone(), + Some(_) => { + errors.push(format!( + "{}: {label} '{}': description must be text", + entry.file, entry.name + )); + continue; + } + }; + entry + .body + .insert("name".to_string(), Value::String(display)); + + match crate::domain::yaml::from_json_with_text::( + Value::Object(entry.body), + Value::Object(entry.text), + ) { + Ok(item) => { + out.insert(entry.name, item); + } + Err(e) => errors.push(format!("{}: {label} '{}': {e}", entry.file, entry.name)), + } + } + out } // Reads and parses a YAML file — fails if it does not exist @@ -1111,19 +1448,6 @@ fn load_yaml_file( Ok(parsed) } -// Reads and parses a YAML file — returns empty HashMap if it does not exist -// `T: DeserializeOwned` is a "trait bound": it says T must be deserializable. -// It's like a type constraint in TypeScript or a Protocol in Python. -fn load_optional_yaml( - path: &Path, -) -> Result, Box> { - if path.exists() { - load_yaml_file(path) - } else { - Ok(HashMap::new()) - } -} - #[cfg(test)] mod tests { use super::*; @@ -1191,6 +1515,25 @@ mod tests { assert!(live.changed_keys(&proposed).is_empty()); } + // server.url is shown back as a link: anything but a web address is a typo. + #[test] + fn server_url_must_be_a_web_address() { + let dir = tempfile::tempdir().unwrap(); + fs::write( + dir.path().join("config.yaml"), + "server:\n host: \"127.0.0.1\"\n port: 9090\n url: \"central.example\"\n", + ) + .unwrap(); + let error = load_config(dir.path().to_str().unwrap()) + .err() + .expect("a bare hostname is refused"); + assert!(error.to_string().contains("server.url"), "{error}"); + let cfg = config_with( + "server:\n host: \"127.0.0.1\"\n port: 9090\n url: \"https://central.example\"\n", + ); + assert_eq!(cfg.server.url.as_deref(), Some("https://central.example")); + } + #[test] fn the_config_api_is_off_unless_it_is_asked_for() { let cfg = config_with("server:\n host: \"127.0.0.1\"\n port: 9090\n"); @@ -1203,17 +1546,248 @@ mod tests { assert!(cfg.config_api.enabled); } - // The list the API serves and the list the loader reads have to be the - // same list, or a file becomes writable that nothing reads (or the other - // way round, which is worse). + // The rule the API applies to what it is asked to write and the rule the + // loader applies to what it finds are one rule, or a file becomes + // writable that nothing reads (or the other way round, which is worse). + #[test] + fn configuration_paths() { + for ok in ["config.yaml", "a/b/c.yml", "sources/acme/src-d42-acme.yaml"] { + assert!(is_config_path(ok), "{ok}"); + } + for bad in [ + "", + "a.json", + "../x.yaml", + "a/../x.yaml", + "/abs.yaml", + "a//b.yaml", + ".hidden.yaml", + "a/.git/x.yaml", + "with space.yaml", + "a\\b.yaml", + "secrets.json", + ] { + assert!(!is_config_path(bad), "{bad:?}"); + } + } + + // ---- item files ------------------------------------------------------- + + fn dir_with(files: &[(&str, &str)]) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + fs::write( + dir.path().join("config.yaml"), + "server:\n host: \"127.0.0.1\"\n port: 9090\n", + ) + .unwrap(); + for (name, body) in files { + let path = dir.path().join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, body).unwrap(); + } + dir + } + + const CONNECTOR: &str = "connectors:\n - name: inv\n project: prj-inv\n path: inv.py\n"; + const PROJECT: &str = + "projects:\n - name: prj-inv\n git_url: https://example.com/inv.git\n"; + + #[test] + fn items_load_from_any_depth_by_their_top_level_key() { + let dir = dir_with(&[ + // Folders are for people: a connector may live anywhere. + ("connectors/inv.yaml", CONNECTOR), + ("projects.yml", PROJECT), + ( + "sources/team-a/storage/src-a.yaml", + "sources:\n - name: src-a\n description: The A inventory\n connector: inv\n ttl_seconds: 60\n", + ), + // Hidden entries are not configuration: the migration's backup, + // the config API's staging, editors, git. + ( + ".legacy-config-20261025T000000Z/sources.yaml", + "nonsense: [", + ), + (".git/HEAD.yaml", "nonsense: ["), + ("sources/.swp.yaml", "nonsense: ["), + // Neither is anything that is not YAML. + ("README.md", "# notes"), + ]); + let cfg = load_config(dir.path().to_str().unwrap()).expect("loads"); + let source = &cfg.sources["src-a"]; + assert_eq!(source.name, "The A inventory"); + assert_eq!(source.connector, "inv"); + assert_eq!(source.script_path, "inv.py"); + assert!(cfg.connectors.contains_key("inv") && cfg.projects.contains_key("prj-inv")); + } + #[test] - fn every_file_the_loader_reads_is_a_known_config_file() { - for name in CONFIG_FILES { - assert!(is_config_file(name)); + fn without_a_description_the_label_is_the_identity() { + let dir = dir_with(&[( + "all.yaml", + &format!( + "{CONNECTOR}{PROJECT}sources:\n - name: src-a\n connector: inv\n ttl_seconds: 60\n" + ), + )]); + let cfg = load_config(dir.path().to_str().unwrap()).expect("loads"); + assert_eq!(cfg.sources["src-a"].name, "src-a"); + assert_eq!(cfg.projects["prj-inv"].name, "prj-inv"); + } + + #[test] + fn a_configmap_mount_of_symlinks_loads() { + // What the kubelet writes: the real files in a hidden folder, and a + // link per key at the top. + let dir = dir_with(&[ + ("..2026_10_25/connectors-inv.yaml", CONNECTOR), + ("..2026_10_25/projects.yaml", PROJECT), + ]); + std::os::unix::fs::symlink(dir.path().join("..2026_10_25"), dir.path().join("..data")) + .unwrap(); + for name in ["connectors-inv.yaml", "projects.yaml"] { + std::os::unix::fs::symlink(format!("..data/{name}"), dir.path().join(name)).unwrap(); + } + let cfg = load_config(dir.path().to_str().unwrap()).expect("loads"); + assert!(cfg.connectors.contains_key("inv")); + } + + #[test] + fn config_maps_take_unquoted_scalars_as_text() { + let dir = dir_with(&[( + "all.yaml", + &format!( + "{CONNECTOR}{PROJECT}sources:\n - name: src-a\n connector: inv\n ttl_seconds: 60\n config:\n concurrency: 20\n insecure: true\n mode: \"0644\"\n" + ), + )]); + let cfg = load_config(dir.path().to_str().unwrap()).expect("loads"); + let config = &cfg.sources["src-a"].config; + assert_eq!(config["concurrency"], "20"); + assert_eq!(config["insecure"], "true"); + assert_eq!(config["mode"], "0644"); + } + + // A text field holds what the file says, not what a number made of it: + // through numbers `1.10` was "1.1", `0x1F` was "31" and a long id a + // float in scientific notation — and a bare number in a list of + // arguments was refused outright. + #[test] + fn text_fields_keep_number_looking_values_verbatim() { + let dir = dir_with(&[( + "all.yaml", + &format!( + "{PROJECT}connectors:\n - name: inv\n project: prj-inv\n path: inv.py\n \ + script_args: [\"--days\", 7, 1.10]\n timeout_seconds: 0x3C\n\ + sources:\n - name: src-a\n description: 2024\n connector: inv\n \ + ttl_seconds: 60\n config:\n API_VERSION: 1.10\n ACCOUNT: 0x1F\n \ + BIG: 123456789012345678901234\n FLAG: True\n PLUS: +3\n\ + enrichers:\n - name: enr-a\n target: src-a\n project: prj-inv\n \ + script_path: e.py\n script_args: [\"--days\", 7]\n" + ), + )]); + let cfg = load_config(dir.path().to_str().unwrap()).expect("loads"); + let source = &cfg.sources["src-a"]; + assert_eq!(source.config["API_VERSION"], "1.10"); + assert_eq!(source.config["ACCOUNT"], "0x1F"); + assert_eq!(source.config["BIG"], "123456789012345678901234"); + assert_eq!(source.config["FLAG"], "True"); + assert_eq!(source.config["PLUS"], "+3"); + assert_eq!(source.name, "2024"); + assert_eq!(source.script_args, ["--days", "7", "1.10"]); + // A field that wants a number still reads one. + assert_eq!(cfg.connectors["inv"].timeout_seconds, 60); + assert_eq!(cfg.enrichers["enr-a"].script_args, ["--days", "7"]); + } + + // 0.34 ids were map keys, always text; a migrated `name: 2024` is the + // item "2024". + #[test] + fn a_number_looking_name_is_the_name() { + let dir = dir_with(&[( + "p.yaml", + "projects:\n - name: 2024\n git_url: https://example.com/inv.git\n", + )]); + let cfg = load_config(dir.path().to_str().unwrap()).expect("loads"); + assert!(cfg.projects.contains_key("2024")); + } + + #[test] + fn duplicate_names_unknown_keys_and_bad_names_are_all_named() { + let dir = dir_with(&[ + ("projects.yaml", PROJECT), + ("connectors/one.yaml", CONNECTOR), + ("connectors/two.yaml", CONNECTOR), + ("typo.yaml", "sorces:\n - name: x\n"), + ("legacy.yaml", "src-a:\n name: A\n"), + ( + "nameless.yaml", + "projects:\n - git_url: https://example.com/x.git\n", + ), + ( + "slash.yaml", + "projects:\n - name: a/b\n git_url: https://example.com/x.git\n", + ), + ("bad field.yaml", "projects: []\n"), + ]); + let err = load_err(&dir); + for expected in [ + "connectors/two.yaml: connector 'inv' is already defined in connectors/one.yaml", + "typo.yaml: unknown key 'sorces'", + "legacy.yaml: unknown key 'src-a'", + "this is the 0.34 layout", + "nameless.yaml: projects item 1 needs a 'name'", + "slash.yaml: projects item 'a/b'", + "bad field.yaml: rename it", + ] { + assert!(err.contains(expected), "missing {expected:?} in {err}"); } - assert!(is_config_file(REQUIRED_CONFIG_FILE)); - assert!(!is_config_file("secrets.yaml")); - assert!(!is_config_file("../etc/passwd")); + } + + #[test] + fn unknown_keys_inside_an_item_name_the_file_and_the_item() { + let dir = dir_with(&[( + "sources/src-a.yaml", + "sources:\n - name: src-a\n connector: remote\n ttl_seconds: 60\n credential_ids: []\n", + )]); + let err = load_err(&dir); + assert!( + err.contains("sources/src-a.yaml: source 'src-a'") && err.contains("credential_ids"), + "{err}" + ); + } + + #[test] + fn connector_rules_are_checked_at_load() { + let dir = dir_with(&[( + "connectors.yaml", + "connectors:\n - name: remote\n path: /opt/x.py\n - name: loose\n path: x.py\n - name: ghostly\n project: prj-ghost\n path: x.py\n", + )]); + let err = load_err(&dir); + assert!( + err.contains("Connector 'remote' uses the name of a built-in"), + "{err}" + ); + assert!( + err.contains("Connector 'loose' has a relative path but no project"), + "{err}" + ); + // A connector's own shape is reported before references are: the + // unknown project is named once the shape is right. + let dir = dir_with(&[( + "connectors.yaml", + "connectors:\n - name: ghostly\n project: prj-ghost\n path: x.py\n", + )]); + assert!( + load_err(&dir).contains("Connector 'ghostly' references unknown project 'prj-ghost'") + ); + } + + #[test] + fn a_source_naming_an_unknown_connector_is_named() { + let dir = dir_with(&[( + "sources.yaml", + "sources:\n - name: src-a\n connector: d43\n ttl_seconds: 60\n", + )]); + assert!(load_err(&dir).contains("Source 'src-a' references unknown connector 'd43'")); } #[test] @@ -1234,7 +1808,7 @@ mod tests { // they stopped being.) fs::write( dir.path().join("credentials.yaml"), - "cred-test:\n name: \"Test\"\n type: \"token\"\n env_prefix: \"TEST\"\n", + "credentials:\n - name: cred-test\n description: \"Test\"\n type: \"token\"\n env_prefix: \"TEST\"\n", ) .unwrap(); @@ -1266,7 +1840,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("enrichers.yaml"), - "enrich-test:\n name: \"Test\"\n target_id: \"src-nonexistent\"\n script_path: \"test.py\"\n", + "enrichers:\n - name: enrich-test\n description: \"Test\"\n target: \"src-nonexistent\"\n script_path: \"test.py\"\n", ).unwrap(); let result = load_config(dir.path().to_str().unwrap()); @@ -1285,7 +1859,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("endpoints.yaml"), - "ep-test:\n name: \"Test\"\n source_ids: [\"src-ghost\"]\n script_path: \"test.py\"\n", + "endpoints:\n - name: ep-test\n description: \"Test\"\n sources: [\"src-ghost\"]\n script_path: \"test.py\"\n", ).unwrap(); let result = load_config(dir.path().to_str().unwrap()); @@ -1304,7 +1878,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("endpoints.yaml"), - "ep-test:\n name: \"Test\"\n", + "endpoints:\n - name: ep-test\n description: \"Test\"\n", ) .unwrap(); @@ -1325,7 +1899,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("endpoints.yaml"), - "ep-test:\n name: \"Test\"\n output: ansible\n script_path: \"x.py\"\n", + "endpoints:\n - name: ep-test\n description: \"Test\"\n output: ansible\n script_path: \"x.py\"\n", ) .unwrap(); @@ -1346,7 +1920,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("endpoints.yaml"), - "ep-ok:\n name: \"Ok\"\n output: json\n config:\n host: \"*.aa1.pqe\"\n group: \"linux\"\n filter: \"os=OracleLinux\"\n fields: \"os\"\n exclude_fields: \"secret\"\n", + "endpoints:\n - name: ep-ok\n description: \"Ok\"\n output: json\n config:\n host: \"*.aa1.pqe\"\n group: \"linux\"\n filter: \"os=OracleLinux\"\n fields: \"os\"\n exclude_fields: \"secret\"\n", ) .unwrap(); load_config(dir.path().to_str().unwrap()).expect("the five shared settings must validate"); @@ -1355,7 +1929,7 @@ mod tests { // unknown key fails listing what a builtin understands. fs::write( dir.path().join("endpoints.yaml"), - "ep-old:\n name: \"Old\"\n output: ansible\n config:\n filter_os: \"OracleLinux\"\n colour: \"red\"\n", + "endpoints:\n - name: ep-old\n description: \"Old\"\n output: ansible\n config:\n filter_os: \"OracleLinux\"\n colour: \"red\"\n", ) .unwrap(); let err = load_config(dir.path().to_str().unwrap()) @@ -1376,7 +1950,7 @@ mod tests { // A malformed static filter fails the load, not every render. fs::write( dir.path().join("endpoints.yaml"), - "ep-bad:\n name: \"Bad\"\n output: csv\n config:\n filter: \"os\"\n", + "endpoints:\n - name: ep-bad\n description: \"Bad\"\n output: csv\n config:\n filter: \"os\"\n", ) .unwrap(); let err = load_config(dir.path().to_str().unwrap()) @@ -1399,7 +1973,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("endpoints.yaml"), - "ep-test:\n name: \"Test\"\n output: ansible\n", + "endpoints:\n - name: ep-test\n description: \"Test\"\n output: ansible\n", ) .unwrap(); @@ -1420,13 +1994,12 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("sources.yaml"), - "src-cmdb:\n name: \"CMDB\"\n project_id: \"prj-test\"\n script_path: \"cmdb.py\"\n ttl_seconds: 60\n\ - src-vmware:\n name: \"VMware\"\n project_id: \"prj-test\"\n script_path: \"vmware.py\"\n ttl_seconds: 60\n", + "sources:\n - name: src-cmdb\n description: \"CMDB\"\n connector: c-src-cmdb\n ttl_seconds: 60\n - name: src-vmware\n description: \"VMware\"\n connector: c-src-vmware\n ttl_seconds: 60\nconnectors:\n - name: c-src-cmdb\n project: \"prj-test\"\n path: \"cmdb.py\"\n - name: c-src-vmware\n project: \"prj-test\"\n path: \"vmware.py\"\n", ) .unwrap(); fs::write(dir.path().join("endpoints.yaml"), endpoints_yaml).unwrap(); @@ -1436,7 +2009,7 @@ mod tests { #[test] fn an_endpoint_limited_by_one_of_its_own_sources_validates() { let dir = config_dir_with_two_sources( - "ep-built:\n name: \"Built\"\n source_ids: [\"src-cmdb\", \"src-vmware\"]\n output: ansible\n limit:\n by_hosts_from_inventory: \"src-cmdb\"\n", + "endpoints:\n - name: ep-built\n description: \"Built\"\n sources: [\"src-cmdb\", \"src-vmware\"]\n output: ansible\n limit:\n by_hosts_from_inventory: \"src-cmdb\"\n", ); let cfg = load_config(dir.path().to_str().unwrap()) @@ -1455,7 +2028,7 @@ mod tests { // src-vmware exists, but this endpoint never merges it -- so the // intersection would be against data the endpoint never waits for. let dir = config_dir_with_two_sources( - "ep-built:\n name: \"Built\"\n source_ids: [\"src-cmdb\"]\n output: ansible\n limit:\n by_hosts_from_inventory: \"src-vmware\"\n", + "endpoints:\n - name: ep-built\n description: \"Built\"\n sources: [\"src-cmdb\"]\n output: ansible\n limit:\n by_hosts_from_inventory: \"src-vmware\"\n", ); let err = load_config(dir.path().to_str().unwrap()) @@ -1463,13 +2036,13 @@ mod tests { .expect("a limit naming a source outside source_ids must fail") .to_string(); assert!(err.contains("src-vmware"), "{}", err); - assert!(err.contains("source_ids"), "{}", err); + assert!(err.contains("add it to sources"), "{}", err); } #[test] fn validate_catches_a_limit_that_names_no_rule() { let dir = config_dir_with_two_sources( - "ep-built:\n name: \"Built\"\n source_ids: [\"src-cmdb\"]\n output: ansible\n limit: {}\n", + "endpoints:\n - name: ep-built\n description: \"Built\"\n sources: [\"src-cmdb\"]\n output: ansible\n limit: {}\n", ); let err = load_config(dir.path().to_str().unwrap()) @@ -1484,7 +2057,7 @@ mod tests { #[test] fn a_typo_inside_the_limit_is_named_at_load() { let dir = config_dir_with_two_sources( - "ep-built:\n name: \"Built\"\n source_ids: [\"src-cmdb\"]\n output: ansible\n limit:\n by_hosts_from_inventary: \"src-cmdb\"\n", + "endpoints:\n - name: ep-built\n description: \"Built\"\n sources: [\"src-cmdb\"]\n output: ansible\n limit:\n by_hosts_from_inventary: \"src-cmdb\"\n", ); let err = load_config(dir.path().to_str().unwrap()) @@ -1506,7 +2079,7 @@ mod tests { // setting it is a mistake to name, like project_id and script_args. fs::write( dir.path().join("endpoints.yaml"), - "ep-test:\n name: \"Test\"\n output: csv\n timeout_seconds: 60\n", + "endpoints:\n - name: ep-test\n description: \"Test\"\n output: csv\n timeout_seconds: 60\n", ) .unwrap(); @@ -1528,7 +2101,7 @@ mod tests { // sources.yaml declares a project_id that does not exist in projects.yaml fs::write( dir.path().join("sources.yaml"), - "src-test:\n name: \"Test\"\n project_id: \"prj-ghost\"\n script_path: \"test.py\"\n ttl_seconds: 60\n", + "sources:\n - name: src-test\n description: \"Test\"\n connector: c-src-test\n ttl_seconds: 60\nconnectors:\n - name: c-src-test\n project: \"prj-ghost\"\n path: \"test.py\"\n", ).unwrap(); let result = load_config(dir.path().to_str().unwrap()); @@ -1547,14 +2120,14 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); let kh_path = dir.path().join("known_hosts"); fs::write( dir.path().join("sources.yaml"), format!( - "src-fleet:\n name: \"Fleet\"\n project_id: \"prj-test\"\n script_path: \"unused\"\n connector_type: \"ssh\"\n ttl_seconds: 60\n config:\n hosts: \"a.example\"\n ssh_known_hosts: \"{}\"\n", + "sources:\n - name: src-fleet\n description: \"Fleet\"\n connector: c-src-fleet\n ttl_seconds: 60\n config:\n hosts: \"a.example\"\n ssh_known_hosts: \"{}\"\nconnectors:\n - name: c-src-fleet\n type: ssh\n path: \"unused\"\n", kh_path.display() ), ) @@ -1581,12 +2154,12 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("sources.yaml"), - "src-script:\n name: \"Script\"\n project_id: \"prj-test\"\n script_path: \"test.py\"\n ttl_seconds: 60\n config:\n ssh_known_hosts: \"/anywhere\"\n", + "sources:\n - name: src-script\n description: \"Script\"\n connector: c-src-script\n ttl_seconds: 60\n config:\n ssh_known_hosts: \"/anywhere\"\nconnectors:\n - name: c-src-script\n project: \"prj-test\"\n path: \"test.py\"\n", ) .unwrap(); @@ -1612,8 +2185,7 @@ mod tests { .err() .map(|e| e.to_string()) }; - let cred = - "cred-ldap:\n name: \"LDAP\"\n type: username_password\n env_prefix: \"LDAP\"\n"; + let cred = "credentials:\n - name: cred-ldap\n description: \"LDAP\"\n type: username_password\n env_prefix: \"LDAP\"\n"; let ldap = |url: &str, extra: &str| { format!( " enabled: true\n ldap:\n url: \"{url}\"\n bind_credential: cred-ldap\n \ @@ -1669,7 +2241,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n credential_id: \"cred-ghost\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n credential: \"cred-ghost\"\n", ).unwrap(); let result = load_config(dir.path().to_str().unwrap()); @@ -1688,12 +2260,12 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("sources.yaml"), - "src-test:\n name: \"Test\"\n project_id: \"prj-test\"\n script_path: \"test.py\"\n ttl_seconds: 60\n", + "sources:\n - name: src-test\n description: \"Test\"\n connector: c-src-test\n ttl_seconds: 60\nconnectors:\n - name: c-src-test\n project: \"prj-test\"\n path: \"test.py\"\n", ).unwrap(); let cfg = load_config(dir.path().to_str().unwrap()).unwrap(); @@ -1710,16 +2282,16 @@ mod tests { .unwrap(); fs::write( dir.path().join("sources.yaml"), - "src-test:\n name: \"Test\"\n project_id: \"prj-test\"\n script_path: \"test.py\"\n ttl_seconds: 60\n", + "sources:\n - name: src-test\n description: \"Test\"\n connector: c-src-test\n ttl_seconds: 60\nconnectors:\n - name: c-src-test\n project: \"prj-test\"\n path: \"test.py\"\n", ).unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("enrichers.yaml"), - "enrich-test:\n name: \"Test\"\n target_id: \"src-test\"\n script_path: \"e.py\"\n project_id: \"prj-ghost\"\n", + "enrichers:\n - name: enrich-test\n description: \"Test\"\n target: \"src-test\"\n script_path: \"e.py\"\n project: \"prj-ghost\"\n", ).unwrap(); let result = load_config(dir.path().to_str().unwrap()); @@ -1737,7 +2309,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); // three violations at once: not an ssh source, self-reference is @@ -1745,12 +2317,13 @@ mod tests { fs::write( dir.path().join("sources.yaml"), concat!( - "src-script:\n name: \"S\"\n project_id: \"prj-test\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", - " hosts_from_source:\n source: \"src-ssh\"\n", - "src-ssh:\n name: \"T\"\n project_id: \"prj-test\"\n script_path: \"gather_facts\"\n ttl_seconds: 60\n", - " connector_type: \"ssh\"\n", - " hosts_from_source:\n source: \"src-ssh\"\n", - " config:\n hosts: \"a.example.com\"\n", + "connectors:\n - name: inv\n project: \"prj-test\"\n path: \"x.py\"\n - name: facts\n type: ssh\n path: \"gather_facts\"\n", + "sources:\n", + " - name: src-script\n connector: inv\n ttl_seconds: 60\n", + " hosts_from_source:\n source: \"src-ssh\"\n", + " - name: src-ssh\n connector: facts\n ttl_seconds: 60\n", + " hosts_from_source:\n source: \"src-ssh\"\n", + " config:\n hosts: \"a.example.com\"\n", ), ) .unwrap(); @@ -1774,16 +2347,16 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("sources.yaml"), concat!( - "src-ssh:\n name: \"S\"\n project_id: \"prj-test\"\n script_path: \"gather_facts\"\n ttl_seconds: 60\n", - " connector_type: \"ssh\"\n", - " host_args: [\"--host\", \"{target}\"]\n", - " config:\n hosts: \"a.example.com\"\n", + "connectors:\n - name: facts\n type: ssh\n path: \"gather_facts\"\n", + " host_args: [\"--host\", \"{target}\"]\n", + "sources:\n - name: src-ssh\n connector: facts\n ttl_seconds: 60\n", + " config:\n hosts: \"a.example.com\"\n", ), ) .unwrap(); @@ -1792,7 +2365,11 @@ mod tests { .err() .expect("expected a validation error") .to_string(); - assert!(err.contains("not a script source"), "missing rule: {}", err); + assert!( + err.contains("not a script connector"), + "missing rule: {}", + err + ); } #[test] @@ -1805,15 +2382,15 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("sources.yaml"), concat!( - "src-ssh:\n name: \"T\"\n project_id: \"prj-test\"\n script_path: \"gather_facts\"\n ttl_seconds: 60\n", - " connector_type: \"ssh\"\n", - " hosts_from_source:\n source: \"src-ghost\"\n", + "connectors:\n - name: facts\n type: ssh\n path: \"gather_facts\"\n", + "sources:\n - name: src-ssh\n connector: facts\n ttl_seconds: 60\n", + " hosts_from_source:\n source: \"src-ghost\"\n", ), ) .unwrap(); @@ -1836,12 +2413,12 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("sources.yaml"), - "src-a:\n name: \"A\"\n project_id: \"prj-test\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", + "sources:\n - name: src-a\n description: \"A\"\n connector: c-src-a\n ttl_seconds: 60\nconnectors:\n - name: c-src-a\n project: \"prj-test\"\n path: \"x.py\"\n", ) .unwrap(); dir @@ -1860,15 +2437,16 @@ mod tests { fs::write( dir.path().join("views.yaml"), concat!( + "views:\n", // id collides with a source, and its member is unknown - "src-a:\n name: \"Collides\"\n members:\n", - " - source: \"src-ghost\"\n owns:\n source: \"src-a\"\n", + " - name: src-a\n description: \"Collides\"\n members:\n", + " - source: \"src-ghost\"\n owns:\n source: \"src-a\"\n", // lists the same member twice, and owns against an unknown source - "vw-dup:\n name: \"Dup\"\n members:\n", - " - source: \"src-a\"\n owns:\n source: \"src-a\"\n", - " - source: \"src-a\"\n owns:\n source: \"src-nowhere\"\n", + " - name: vw-dup\n description: \"Dup\"\n members:\n", + " - source: \"src-a\"\n owns:\n source: \"src-a\"\n", + " - source: \"src-a\"\n owns:\n source: \"src-nowhere\"\n", // no members at all - "vw-empty:\n name: \"Empty\"\n members: []\n", + " - name: vw-empty\n description: \"Empty\"\n members: []\n", ), ) .unwrap(); @@ -1887,10 +2465,11 @@ mod tests { fs::write( dir.path().join("views.yaml"), concat!( - "vw-inner:\n name: \"Inner\"\n members:\n", - " - source: \"src-a\"\n owns:\n source: \"src-a\"\n", - "vw-outer:\n name: \"Outer\"\n members:\n", - " - source: \"vw-inner\"\n owns:\n source: \"src-a\"\n", + "views:\n", + " - name: vw-inner\n description: \"Inner\"\n members:\n", + " - source: \"src-a\"\n owns:\n source: \"src-a\"\n", + " - name: vw-outer\n description: \"Outer\"\n members:\n", + " - source: \"vw-inner\"\n owns:\n source: \"src-a\"\n", ), ) .unwrap(); @@ -1903,17 +2482,17 @@ mod tests { let dir = dir_with_one_source(); fs::write( dir.path().join("views.yaml"), - "vw-a:\n name: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n", + "views:\n - name: vw-a\n description: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n", ) .unwrap(); fs::write( dir.path().join("endpoints.yaml"), - "ep-a:\n name: \"E\"\n source_ids: [\"vw-a\"]\n script_path: \"t.py\"\n", + "endpoints:\n - name: ep-a\n description: \"E\"\n sources: [\"vw-a\"]\n script_path: \"t.py\"\n", ) .unwrap(); fs::write( dir.path().join("enrichers.yaml"), - "en-a:\n name: \"E\"\n target_id: \"vw-a\"\n script_path: \"t.py\"\n", + "enrichers:\n - name: en-a\n description: \"E\"\n target: \"vw-a\"\n script_path: \"t.py\"\n", ) .unwrap(); @@ -1927,12 +2506,12 @@ mod tests { let dir = dir_with_one_source(); fs::write( dir.path().join("views.yaml"), - "vw-a:\n name: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n", + "views:\n - name: vw-a\n description: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n", ) .unwrap(); fs::write( dir.path().join("api_keys.yaml"), - "key-forms:\n name: \"Forms\"\n env: \"X\"\n sources: [\"vw-a\"]\n", + "api_keys:\n - name: key-forms\n description: \"Forms\"\n env: \"X\"\n sources: [\"vw-a\"]\n", ) .unwrap(); @@ -1945,7 +2524,7 @@ mod tests { let dir = dir_with_one_source(); fs::write( dir.path().join("views.yaml"), - "vw-a:\n name: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n grups: [\"x\"]\n", + "views:\n - name: vw-a\n description: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n grups: [\"x\"]\n", ) .unwrap(); @@ -1967,14 +2546,14 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); // sync_interval_second (no s): the sync interval this operator thinks // they configured would silently not exist fs::write( dir.path().join("sources.yaml"), - "src-test:\n name: \"Test\"\n project_id: \"prj-test\"\n script_path: \"x.py\"\n ttl_seconds: 60\n sync_interval_second: 300\n", + "sources:\n - name: src-test\n description: \"Test\"\n connector: c-src-test\n ttl_seconds: 60\n sync_interval_second: 300\nconnectors:\n - name: c-src-test\n project: \"prj-test\"\n path: \"x.py\"\n", ) .unwrap(); @@ -1992,7 +2571,7 @@ mod tests { // carry would silently become an empty one fs::write( dir.path().join("api_keys.yaml"), - "key-forms:\n name: \"Forms\"\n env: \"X\"\n source: [\"src-a\"]\n", + "api_keys:\n - name: key-forms\n description: \"Forms\"\n env: \"X\"\n source: [\"src-a\"]\n", ) .unwrap(); @@ -2040,7 +2619,7 @@ mod tests { // ownership with no fallback can never route anything fs::write( dir.path().join("views.yaml"), - "vw-a:\n name: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n advertised: true\n", + "views:\n - name: vw-a\n description: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n advertised: true\n", ) .unwrap(); assert!(load_err(&dir).contains("claim nothing")); @@ -2048,7 +2627,7 @@ mod tests { // A declared fallback makes it valid fs::write( dir.path().join("views.yaml"), - "vw-a:\n name: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n advertised: true\n groups: [\"dc1\"]\n", + "views:\n - name: vw-a\n description: \"V\"\n members:\n - source: \"src-a\"\n owns:\n source: \"src-a\"\n advertised: true\n groups: [\"dc1\"]\n", ) .unwrap(); load_config(dir.path().to_str().unwrap()).expect("fallback makes it valid"); @@ -2059,7 +2638,7 @@ mod tests { let dir = dir_with_one_source(); fs::write( dir.path().join("sources.yaml"), - "src-a:\n name: \"A\"\n project_id: \"prj-test\"\n script_path: \"x.py\"\n ttl_seconds: 60\n advertise_scope: {}\n", + "sources:\n - name: src-a\n description: \"A\"\n connector: c-src-a\n ttl_seconds: 60\n advertise_scope: {}\nconnectors:\n - name: c-src-a\n project: \"prj-test\"\n path: \"x.py\"\n", ) .unwrap(); @@ -2078,18 +2657,20 @@ mod tests { .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", ) .unwrap(); fs::write( dir.path().join("sources.yaml"), concat!( + "connectors:\n - name: inv\n project: \"prj-test\"\n path: \"x.py\"\n", + "sources:\n", // junk in the schedule - "src-bad:\n name: \"B\"\n project_id: \"prj-test\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", - " schedule: \"whenever feels right\"\n", + " - name: src-bad\n connector: inv\n ttl_seconds: 60\n", + " schedule: \"whenever feels right\"\n", // schedule AND interval - "src-both:\n name: \"C\"\n project_id: \"prj-test\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", - " schedule: \"0 2 * * *\"\n sync_interval_seconds: 60\n", + " - name: src-both\n connector: inv\n ttl_seconds: 60\n", + " schedule: \"0 2 * * *\"\n sync_interval_seconds: 60\n", ), ) .unwrap(); @@ -2098,15 +2679,16 @@ mod tests { // concept, whatever kind of periodic work it paces fs::write( dir.path().join("enrichers.yaml"), - "en-bad:\n name: \"E\"\n target_id: \"src-both\"\n script_path: \"e.py\"\n schedule: \"nope\"\n", + "enrichers:\n - name: en-bad\n description: \"E\"\n target: \"src-both\"\n script_path: \"e.py\"\n schedule: \"nope\"\n", ) .unwrap(); fs::write( dir.path().join("projects.yaml"), concat!( - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n", - "prj-both:\n name: \"P\"\n git_url: \"https://example.com/p.git\"\n", - " schedule: \"0 2 * * *\"\n sync_interval_seconds: 60\n", + "projects:\n", + " - name: prj-test\n git_url: \"https://example.com/repo.git\"\n", + " - name: prj-both\n git_url: \"https://example.com/p.git\"\n", + " schedule: \"0 2 * * *\"\n sync_interval_seconds: 60\n", ), ) .unwrap(); @@ -2124,17 +2706,17 @@ mod tests { // Well-formed cron-only definitions of every kind load fs::write( dir.path().join("sources.yaml"), - "src-cron:\n name: \"OK\"\n project_id: \"prj-test\"\n script_path: \"x.py\"\n ttl_seconds: 60\n schedule: \"30 2 * * *\"\n", + "sources:\n - name: src-cron\n description: \"OK\"\n connector: c-src-cron\n ttl_seconds: 60\n schedule: \"30 2 * * *\"\nconnectors:\n - name: c-src-cron\n project: \"prj-test\"\n path: \"x.py\"\n", ) .unwrap(); fs::write( dir.path().join("enrichers.yaml"), - "en-cron:\n name: \"E\"\n target_id: \"src-cron\"\n script_path: \"e.py\"\n schedule: \"45 2 * * *\"\n", + "enrichers:\n - name: en-cron\n description: \"E\"\n target: \"src-cron\"\n script_path: \"e.py\"\n schedule: \"45 2 * * *\"\n", ) .unwrap(); fs::write( dir.path().join("projects.yaml"), - "prj-test:\n name: \"Test\"\n git_url: \"https://example.com/repo.git\"\n schedule: \"15 2 * * *\"\n", + "projects:\n - name: prj-test\n description: \"Test\"\n git_url: \"https://example.com/repo.git\"\n schedule: \"15 2 * * *\"\n", ) .unwrap(); load_config(dir.path().to_str().unwrap()).expect("a valid cron schedule loads"); @@ -2150,7 +2732,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("credentials.yaml"), - "cred-v:\n name: \"V\"\n type: \"token\"\n vault_path: \"team/api\"\n", + "credentials:\n - name: cred-v\n description: \"V\"\n type: \"token\"\n vault_path: \"team/api\"\n", ) .unwrap(); @@ -2179,7 +2761,7 @@ mod tests { .unwrap(); fs::write( dir.path().join("sources.yaml"), - "src-test:\n name: \"Test\"\n project_id: \"p\"\n script_path: \"test.py\"\n credential_ids: [\"cred-missing\"]\n ttl_seconds: 60\n", + "sources:\n - name: src-test\n description: \"Test\"\n connector: c-src-test\n credentials: [\"cred-missing\"]\n ttl_seconds: 60\nconnectors:\n - name: c-src-test\n project: \"p\"\n path: \"test.py\"\n", ).unwrap(); let result = load_config(dir.path().to_str().unwrap()); diff --git a/src/config/migrate.rs b/src/config/migrate.rs new file mode 100644 index 0000000..c019f52 --- /dev/null +++ b/src/config/migrate.rs @@ -0,0 +1,1507 @@ +// The one-time conversion of a 0.34 configuration directory into the 0.35 +// layout. +// +// Until 0.35 the directory was eight fixed files, each an id-keyed map: +// +// # sources.yaml +// src-d42: +// name: "Device42" +// connector_type: script +// project_id: prj-inventory +// script_path: dynamic/d42/inventory.py +// ... +// +// From 0.35 every item is a list entry under a typed key, `name` is the +// identity, references lose their `_id` suffix, and what a source used to say +// about its SCRIPT moved to a connector item. A pod upgraded in place still +// has the old files on its volume, and refusing to start on them would brick +// it — so startup converts them, once, before loading: +// +// - detect: a legacy file name at the root whose top level is a map of maps +// (a 0.35 file has lists under its keys, never maps); +// - plan: compute every new file in memory, connectors extracted from the +// sources and de-duplicated; +// - validate: write the plan into a hidden staging folder beside the real +// files and load it exactly as startup would — nothing moves if it fails; +// - commit: move the old files into `.legacy-config-/` (hidden, +// so the loader skips it) and rename the new ones into place. +// +// Comments in the old files cannot survive: they are not data. The backup +// keeps them, next to the new files, for whoever wants to carry them over. + +use std::collections::{BTreeMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde_json::{Map, Value}; + +use super::{REQUIRED_CONFIG_FILE, config_files, is_safe_id, load_config_detailed}; + +// The 0.34 files and the section each becomes. +pub const LEGACY_FILES: [(&str, &str); 7] = [ + ("credentials.yaml", "credentials"), + ("projects.yaml", "projects"), + ("sources.yaml", "sources"), + ("views.yaml", "views"), + ("enrichers.yaml", "enrichers"), + ("endpoints.yaml", "endpoints"), + ("api_keys.yaml", "api_keys"), +]; + +// Reference keys that dropped their `_id` suffix, per section. +const RENAMES: [(&str, &str, &str); 7] = [ + ("projects", "credential_id", "credential"), + ("enrichers", "target_id", "target"), + ("enrichers", "source_id", "source"), + ("enrichers", "project_id", "project"), + ("endpoints", "source_ids", "sources"), + ("endpoints", "project_id", "project"), + ("sources", "credential_ids", "credentials"), +]; + +// What a source carried about its script, which now belongs to a connector. +const CONNECTOR_KEYS: [&str; 6] = [ + "connector_type", + "project_id", + "script_path", + "script_args", + "host_args", + "output_format", +]; + +// Key order in the files written, per section: identity and label first, +// then references, then the rest in the order the docs describe them. Keys +// not listed (a typo the old loader would have refused too) follow, +// alphabetically, so the new loader names them. +const ORDER: [(&str, &[&str]); 8] = [ + ( + "sources", + &[ + "connector", + "credentials", + "project", + "path", + "timeout_seconds", + "sync_mode", + "schedule", + "sync_interval_seconds", + "ttl_seconds", + "ttl_overrides", + "allow_on_demand_refresh", + "host_sync_updates_group_vars", + "hosts_from_source", + "advertise_scope", + "config", + ], + ), + ( + "connectors", + &[ + "type", + "project", + "path", + "script_args", + "host_args", + "output_format", + "timeout_seconds", + ], + ), + ("views", &["members", "ttl_seconds"]), + ( + "endpoints", + &[ + "sources", + "output", + "project", + "script_path", + "script_args", + "timeout_seconds", + "limit", + "config", + ], + ), + ( + "enrichers", + &[ + "target", + "source", + "project", + "script_path", + "script_args", + "fields", + "fields_excluded", + "groups", + "groups_excluded", + "schedule", + "sync_interval_seconds", + "timeout_seconds", + "config", + ], + ), + ( + "projects", + &[ + "git_url", + "branch", + "credential", + "sync_on_boot", + "schedule", + "sync_interval_seconds", + "timeout_seconds", + "python_venv", + ], + ), + ( + "credentials", + &[ + "type", + "vault_path", + "env_prefix", + "secret_file", + "secret_keys", + "file_keys", + ], + ), + ("api_keys", &["env", "role", "sources", "endpoints"]), +]; + +// The new layout, computed and not yet written. +#[derive(Debug, Default)] +pub struct MigrationPlan { + // The legacy files found at the root (they move to the backup). + pub legacy_files: Vec, + // Relative path → contents, one file per item: `
/.yaml`. + pub files: BTreeMap, + // Each generated connector and the sources that use it. + pub connectors: BTreeMap>, + // Anything worth a line in the log that the files themselves do not say. + pub notes: Vec, +} + +// What a migration did, for the log. +#[derive(Debug)] +pub struct MigrationOutcome { + pub plan: MigrationPlan, + pub backup_dir: PathBuf, +} + +impl MigrationPlan { + // One line per fact, for the startup log and `--migrate-config`. + pub fn summary(&self) -> Vec { + let mut lines = vec![format!( + "converted {} into {} item files", + self.legacy_files.join(", "), + self.files.len() + )]; + for (connector, users) in &self.connectors { + lines.push(format!( + "connector '{}' extracted from: {}", + connector, + users.join(", ") + )); + } + lines.extend(self.notes.iter().cloned()); + lines + } +} + +// The legacy files present at the root of `dir`, if the directory is in the +// 0.34 layout; empty when it is not. An error when a legacy-named file is in +// neither shape, which is the one case where guessing would be wrong. +pub fn detect(dir: &Path) -> Result, String> { + let mut legacy = Vec::new(); + let mut empty = Vec::new(); + for (file, _) in LEGACY_FILES { + let path = dir.join(file); + if !path.is_file() { + continue; + } + let text = fs::read_to_string(&path).map_err(|e| format!("{file}: {e}"))?; + match shape(&text).map_err(|e| format!("{file}: {e}"))? { + Shape::Legacy => legacy.push(file.to_string()), + Shape::Empty => empty.push(file.to_string()), + Shape::Items => {} + } + } + // An empty legacy file is moved with the others (it is part of what the + // old directory was), but on its own it says nothing about the layout. + if !legacy.is_empty() { + legacy.extend(empty); + legacy.sort(); + } + Ok(legacy) +} + +enum Shape { + Legacy, + Items, + Empty, +} + +// A 0.35 file holds lists under its top-level keys; a 0.34 file holds maps +// under ids. Nothing written for one layout reads as the other. +fn shape(text: &str) -> Result { + match crate::domain::yaml::to_json(text)? { + Value::Null => Ok(Shape::Empty), + Value::Object(top) if top.is_empty() => Ok(Shape::Empty), + Value::Object(top) => { + let maps = top.values().filter(|v| v.is_object()).count(); + if maps == top.len() { + Ok(Shape::Legacy) + } else if maps == 0 { + Ok(Shape::Items) + } else { + Err( + "mixes id-keyed items (the 0.34 layout) with lists (0.35) — \ + convert it by hand" + .to_string(), + ) + } + } + _ => Ok(Shape::Items), + } +} + +// Compute the new layout for `dir`, without writing anything. None when the +// directory is not in the old layout. +pub fn plan(dir: &Path) -> Result, String> { + let legacy = detect(dir)?; + if legacy.is_empty() { + return Ok(None); + } + let mut plan = MigrationPlan { + legacy_files: legacy.clone(), + ..MigrationPlan::default() + }; + + // Section → id → item body (already renamed), in file order. + let mut sections: BTreeMap<&str, BTreeMap>> = BTreeMap::new(); + for (file, section) in LEGACY_FILES { + if !legacy.iter().any(|f| f == file) { + continue; + } + let text = fs::read_to_string(dir.join(file)).map_err(|e| format!("{file}: {e}"))?; + let typed = crate::domain::yaml::to_json(&text)?; + let verbatim = crate::domain::yaml::to_text_json(&text)?; + let Value::Object(items) = keep_plain_text(typed, verbatim) else { + continue; + }; + for (id, body) in items { + let Value::Object(body) = body else { + return Err(format!("{file}: '{id}' is not a mapping")); + }; + if !is_safe_id(&id) { + return Err(format!( + "{file}: '{id}' cannot be an item name — names may only use letters, \ + digits, '-', '_' and '.'" + )); + } + sections + .entry(section) + .or_default() + .insert(id, renamed(section, body)); + } + } + + // Sources: split each into the source and its connector. + let sources = sections.remove("sources").unwrap_or_default(); + let (sources, connectors) = extract_connectors(sources, &mut plan)?; + sections.insert("sources", sources); + sections.insert("connectors", connectors); + + for (section, items) in §ions { + for (id, body) in items { + let path = format!("{section}/{id}.yaml"); + plan.files.insert(path, item_file(section, id, body)); + } + } + Ok(Some(plan)) +} + +// `name` → `description`, the `_id` references renamed. +fn renamed(section: &str, mut body: Map) -> Map { + if let Some(name) = body.remove("name") { + body.insert("description".to_string(), name); + } + for (s, old, new) in RENAMES { + if s == section + && let Some(value) = body.remove(old) + { + body.insert(new.to_string(), value); + } + } + body +} + +// A plain scalar whose text is not how its value prints (`1.10`, `0x1F`, +// `True`, `+3`), carried through the plan as `{PLAIN: ""}` and +// written back as that same text. As a number it would be written `1.1` or +// `31`: a `config:` value or a script argument changed on disk, with the +// original only in the backup. Written verbatim, it reads back exactly as the +// 0.34 file did — as text where a field wants text, as the number where one +// wants a number. The key starts with a NUL, which no configuration key has. +const PLAIN: &str = "\u{0}plain"; + +fn keep_plain_text(typed: Value, text: Value) -> Value { + match (typed, text) { + (Value::Object(typed), Value::Object(mut text)) => Value::Object( + typed + .into_iter() + .map(|(k, v)| { + let t = text.remove(&k).unwrap_or(Value::Null); + (k, keep_plain_text(v, t)) + }) + .collect(), + ), + (Value::Array(typed), Value::Array(text)) => Value::Array( + typed + .into_iter() + .zip(text.into_iter().chain(std::iter::repeat(Value::Null))) + .map(|(v, t)| keep_plain_text(v, t)) + .collect(), + ), + (value @ (Value::Number(_) | Value::Bool(_)), Value::String(text)) => { + let printed = value.to_string(); + if printed == text { + value + } else { + Value::Object(Map::from_iter([(PLAIN.to_string(), Value::String(text))])) + } + } + (value, _) => value, + } +} + +// The text a carried scalar was written as (see PLAIN). +fn plain(value: &Value) -> Option<&str> { + match value { + Value::Object(map) if map.len() == 1 => map.get(PLAIN).and_then(Value::as_str), + _ => None, + } +} + +// A value as text: a string, or a scalar's own text. +fn as_text(value: &Value) -> Option { + match value { + Value::String(s) => Some(s.clone()), + Value::Number(_) | Value::Bool(_) => Some(value.to_string()), + other => plain(other).map(str::to_string), + } +} + +// A whole number, however it was written (`600`, `0x258`, `+600`). +fn as_u64(value: &Value) -> Option { + match plain(value) { + Some(text) => crate::domain::yaml::to_json(text).ok()?.as_u64(), + None => value.as_u64(), + } +} + +type Items = BTreeMap>; + +// What makes two sources share a connector: everything a connector holds +// except its timeout, which a source may override. +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone)] +struct ConnectorKey { + kind: String, + project: Option, + script_path: String, + script_args: Vec, + host_args: Vec, + output_format: Option, +} + +fn extract_connectors(sources: Items, plan: &mut MigrationPlan) -> Result<(Items, Items), String> { + let mut out = Items::new(); + // key → (source id, timeout) of every source using it, in id order + let mut groups: BTreeMap> = BTreeMap::new(); + let default_timeout = crate::domain::default_timeout_seconds(); + + for (id, mut body) in sources { + let text = |body: &Map, key: &str| -> Option { + body.get(key).and_then(as_text) + }; + let list = |body: &Map, key: &str| -> Vec { + body.get(key) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .map(|v| as_text(v).unwrap_or_else(|| v.to_string())) + .collect() + }) + .unwrap_or_default() + }; + let kind = text(&body, "connector_type").unwrap_or_else(|| "script".to_string()); + let project = text(&body, "project_id"); + let script_path = text(&body, "script_path").unwrap_or_default(); + let script_args = list(&body, "script_args"); + let host_args = list(&body, "host_args"); + let output_format = text(&body, "output_format"); + for key in CONNECTOR_KEYS { + body.remove(key); + } + + match kind.as_str() { + "remote" => { + // script_path was the source id on the remote instance; it + // now sits in config beside the URL. project_id was required + // and never used. + let config = body + .entry("config") + .or_insert_with(|| Value::Object(Map::new())); + let Value::Object(config) = config else { + return Err(format!( + "sources.yaml: '{id}' has a config that is not a map" + )); + }; + if config.contains_key("source") { + return Err(format!( + "sources.yaml: '{id}' is a remote source whose config already has a \ + 'source' key, which 0.35 uses for the remote source id" + )); + } + config.insert("source".to_string(), Value::String(script_path)); + body.insert("connector".to_string(), Value::String("remote".into())); + note_ignored(plan, &id, &script_args, &host_args, &output_format); + } + "static_inventory" => { + body.insert( + "connector".to_string(), + Value::String("static_inventory".into()), + ); + if let Some(project) = project { + body.insert("project".to_string(), Value::String(project)); + } + body.insert("path".to_string(), Value::String(script_path)); + note_ignored(plan, &id, &script_args, &host_args, &output_format); + } + "script" | "ssh" => { + let ssh = kind == "ssh"; + if ssh && project.is_some() { + // Required in 0.34 and never used: an ssh source runs its + // script_path on the remote host. + plan.notes.push(format!( + "source '{id}': its project_id was dropped — an ssh connector \ + runs nothing from a checkout" + )); + } + let key = ConnectorKey { + kind: kind.clone(), + project: if ssh { None } else { project }, + script_path, + script_args, + host_args: if ssh { Vec::new() } else { host_args }, + // native is the default: writing it out would make two + // sources that only differ in spelling it look different + output_format: if ssh { + None + } else { + output_format.filter(|f| f != "native") + }, + }; + let timeout = body + .get("timeout_seconds") + .and_then(as_u64) + .unwrap_or(default_timeout); + groups.entry(key).or_default().push((id.clone(), timeout)); + } + other => { + return Err(format!( + "sources.yaml: '{id}' has an unknown connector_type '{other}'" + )); + } + } + out.insert(id, body); + } + + // Name each connector, then point its sources at it. + let names = connector_names(groups.keys()); + let mut connectors = Items::new(); + for ((key, users), name) in groups.into_iter().zip(names) { + // The timeout most of its sources used is the connector's; a source + // that differs keeps its own as an override. + let mut tally: BTreeMap = BTreeMap::new(); + for (_, t) in &users { + *tally.entry(*t).or_default() += 1; + } + let timeout = tally + .iter() + .max_by_key(|(t, n)| (**n, **t)) + .map(|(t, _)| *t) + .unwrap_or(default_timeout); + + let mut body = Map::new(); + body.insert("type".to_string(), Value::String(key.kind.clone())); + if let Some(project) = &key.project { + body.insert("project".to_string(), Value::String(project.clone())); + } + // `path`, as on every item where `project` + `path` name a file in a + // checkout; a source's 0.34 `script_path` is that file. + body.insert("path".to_string(), Value::String(key.script_path.clone())); + if !key.script_args.is_empty() { + body.insert("script_args".to_string(), strings(&key.script_args)); + } + if !key.host_args.is_empty() { + body.insert("host_args".to_string(), strings(&key.host_args)); + } + if let Some(format) = &key.output_format { + body.insert("output_format".to_string(), Value::String(format.clone())); + } + if timeout != default_timeout { + body.insert("timeout_seconds".to_string(), Value::from(timeout)); + } + + for (source_id, t) in &users { + let source = out.get_mut(source_id).expect("grouped from out"); + source.insert("connector".to_string(), Value::String(name.clone())); + if *t == timeout { + source.remove("timeout_seconds"); + } else { + source.insert("timeout_seconds".to_string(), Value::from(*t)); + } + } + plan.connectors.insert( + name.clone(), + users.iter().map(|(id, _)| id.clone()).collect(), + ); + connectors.insert(name, body); + } + Ok((out, connectors)) +} + +// One distinct name per connector, in `keys` order. Each takes its base name +// when it is the first to want it and it is free; the rest take the lowest +// free `-N`. Free means: not a built-in connector's name (a script in a +// folder called `remote` would otherwise be refused by validation), and not +// any name already handed out — counting repeats of one base was not enough, +// as the second `foo.py` became `foo-2` while a script `foo-2.py` was also +// `foo-2`, and one connector silently replaced the other. +fn connector_names<'k>(keys: impl Iterator) -> Vec { + let bases: Vec = keys.map(connector_base_name).collect(); + let mut used: HashSet = crate::domain::connector::BUILT_IN + .iter() + .map(|b| b.to_string()) + .collect(); + let mut names: Vec> = bases + .iter() + .map(|base| used.insert(base.clone()).then(|| base.clone())) + .collect(); + for (name, base) in names.iter_mut().zip(&bases) { + if name.is_none() { + let free = (2..) + .map(|n| format!("{base}-{n}")) + .find(|candidate| !used.contains(candidate)) + .expect("an unbounded range finds a free name"); + used.insert(free.clone()); + *name = Some(free); + } + } + names.into_iter().flatten().collect() +} + +fn note_ignored( + plan: &mut MigrationPlan, + id: &str, + script_args: &[String], + host_args: &[String], + output_format: &Option, +) { + if !script_args.is_empty() || !host_args.is_empty() || output_format.is_some() { + plan.notes.push(format!( + "source '{id}': script_args/host_args/output_format were dropped — its \ + built-in connector never used them" + )); + } +} + +fn strings(items: &[String]) -> Value { + Value::Array(items.iter().cloned().map(Value::String).collect()) +} + +// A readable name for a generated connector: the script's own name, or its +// folder when the file is called something generic (`d42/inventory.py` is +// "d42"); `ssh-` for an ssh connector with a one-word path, +// `ssh-script` for one that carries a whole remote script. +fn connector_base_name(key: &ConnectorKey) -> String { + let path = key.script_path.trim(); + if key.kind == "ssh" { + return if is_safe_id(path) && !path.contains('.') { + format!("ssh-{}", path.replace('_', "-")) + } else { + "ssh-script".to_string() + }; + } + let p = Path::new(path); + let stem = p + .file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + let generic = ["inventory", "main", "index", "run", "script"]; + let chosen = if generic.iter().any(|g| stem.starts_with(g)) { + p.parent() + .and_then(|d| d.file_name()) + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or(stem) + } else { + stem + }; + let cleaned: String = chosen + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '-' | '_') { + c + } else { + '-' + } + }) + .collect(); + let cleaned = cleaned.trim_matches('-').to_string(); + if cleaned.is_empty() { + "connector".to_string() + } else { + cleaned + } +} + +// One item file: `---`, the section key, the item with `name` first. +fn item_file(section: &str, id: &str, body: &Map) -> String { + let order: &[&str] = ORDER + .iter() + .find(|(s, _)| *s == section) + .map(|(_, o)| *o) + .unwrap_or(&[]); + let mut keys: Vec<&String> = Vec::new(); + for wanted in std::iter::once(&"description").chain(order.iter()) { + if let Some((k, _)) = body.get_key_value(*wanted) { + keys.push(k); + } + } + let mut rest: Vec<&String> = body + .keys() + .filter(|k| k.as_str() != "description" && !order.contains(&k.as_str())) + .collect(); + rest.sort(); + keys.extend(rest); + + let mut out = format!("---\n{section}:\n - name: {}\n", scalar_key(id)); + for key in keys { + emit_entry(&mut out, 4, key, &body[key]); + } + out +} + +// A minimal YAML writer: block mappings, flow lists of scalars, block lists +// of mappings, strings double-quoted (a JSON string is a valid YAML +// double-quoted scalar) or as a literal block when they span lines. Enough +// for configuration, and what it writes reads back through domain::yaml as +// the same values. +fn emit_entry(out: &mut String, indent: usize, key: &str, value: &Value) { + let pad = " ".repeat(indent); + match value { + _ if plain(value).is_some() => { + out.push_str(&format!("{pad}{}: {}\n", scalar_key(key), inline(value))); + } + Value::Object(map) if !map.is_empty() => { + out.push_str(&format!("{pad}{}:\n", scalar_key(key))); + for (k, v) in nested_order(map) { + emit_entry(out, indent + 2, k, v); + } + } + // Lists of mappings, and lists too long to read on one line, as block + // lists: an endpoint's eight sources read better one per line. + Value::Array(items) + if items + .iter() + .any(|v| (v.is_object() && plain(v).is_none()) || v.is_array()) + || indent + key.len() + inline(value).len() > 96 => + { + out.push_str(&format!("{pad}{}:\n", scalar_key(key))); + for item in items { + emit_list_item(out, indent + 2, item); + } + } + Value::String(s) if literal_block(s) => { + out.push_str(&format!( + "{pad}{}: {}\n", + scalar_key(key), + block(s, indent + 2) + )); + } + other => { + out.push_str(&format!("{pad}{}: {}\n", scalar_key(key), inline(other))); + } + } +} + +fn emit_list_item(out: &mut String, indent: usize, item: &Value) { + let pad = " ".repeat(indent); + match item { + Value::Object(map) if !map.is_empty() && plain(item).is_none() => { + let mut first = true; + for (k, v) in nested_order(map) { + let mut entry = String::new(); + emit_entry(&mut entry, indent + 2, k, v); + if first { + // " key: v" → " - key: v" + entry.replace_range(indent..indent + 2, "- "); + first = false; + } + out.push_str(&entry); + } + } + other => out.push_str(&format!("{pad}- {}\n", inline(other))), + } +} + +// Inside an item, what a nested mapping is ABOUT goes first (a view member's +// `source`, a remote's `url`); the rest stays alphabetical. +fn nested_order(map: &Map) -> Vec<(&String, &Value)> { + const FIRST: [&str; 3] = ["source", "url", "name"]; + let mut entries: Vec<(&String, &Value)> = map.iter().collect(); + entries.sort_by_key(|(k, _)| { + ( + FIRST.iter().position(|f| f == k).unwrap_or(FIRST.len()), + k.to_string(), + ) + }); + entries +} + +fn inline(value: &Value) -> String { + if let Some(text) = plain(value) { + return text.to_string(); + } + match value { + Value::Null => "null".to_string(), + Value::Object(map) if map.is_empty() => "{}".to_string(), + // Flow style: `["--list"]`, which is how the files were written by hand. + Value::Array(items) => format!( + "[{}]", + items.iter().map(inline).collect::>().join(", ") + ), + Value::String(s) => serde_json::to_string(s).unwrap_or_default(), + other => other.to_string(), + } +} + +// A multi-line string reads better as a literal block — a remote script was +// written that way. Leading whitespace or control characters other than the +// newlines would need an indentation indicator or escapes, so those stay +// double-quoted, which is always exact. +fn literal_block(s: &str) -> bool { + s.contains('\n') + && !s.starts_with([' ', '\t', '\n']) + && !s.chars().any(|c| c.is_control() && c != '\n') + && s.lines().all(|l| l.trim_end() == l) +} + +fn block(s: &str, indent: usize) -> String { + let pad = " ".repeat(indent); + let body = s.trim_end_matches('\n'); + let trailing = s.len() - body.len(); + let chomp = match trailing { + 0 => "|-", + 1 => "|", + _ => "|+", + }; + let mut out = chomp.to_string(); + for line in body.split('\n') { + out.push('\n'); + if !line.is_empty() { + out.push_str(&pad); + out.push_str(line); + } + } + for _ in 1..trailing { + out.push('\n'); + } + out +} + +fn scalar_key(key: &str) -> String { + let plain = !key.is_empty() + && key + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) + && !key.starts_with(['-', '.']) + && !matches!( + key.to_ascii_lowercase().as_str(), + "true" | "false" | "null" | "~" | "yes" | "no" | "on" | "off" + ); + if plain { + key.to_string() + } else { + serde_json::to_string(key).unwrap_or_default() + } +} + +fn timestamp() -> String { + chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string() +} + +// `--`: a temporary folder name no other check or copy in +// this process (the tests run many at once) or another one can be using. +fn unique_suffix() -> String { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + format!("{}-{}-{n}", std::process::id(), timestamp()) +} + +// Load the planned layout as startup would, in a scratch copy of the +// directory: config.yaml, every 0.35 file already there, and the plan. +// `scratch` must be new (or empty); it is left for the caller to remove. +fn validate_in(dir: &Path, plan: &MigrationPlan, scratch: &Path) -> Result<(), String> { + let mut files: BTreeMap = BTreeMap::new(); + for name in config_files(dir)? { + if plan.legacy_files.contains(&name) { + continue; + } + let text = fs::read_to_string(dir.join(&name)).map_err(|e| format!("{name}: {e}"))?; + files.insert(name, text); + } + for (name, text) in &plan.files { + if files.contains_key(name) { + return Err(format!( + "{name} already exists — the migration will not overwrite it; move it \ + aside and start again" + )); + } + files.insert(name.clone(), text.clone()); + } + if !files.contains_key(REQUIRED_CONFIG_FILE) { + return Err(format!("{REQUIRED_CONFIG_FILE} is missing")); + } + write_tree(scratch, &files)?; + load_config_detailed(scratch) + .map(|_| ()) + .map_err(|e| format!("the converted configuration does not load: {e}")) +} + +fn write_tree(root: &Path, files: &BTreeMap) -> Result<(), String> { + for (name, text) in files { + let path = root.join(name); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; + } + fs::write(&path, text).map_err(|e| format!("write {}: {e}", path.display()))?; + } + Ok(()) +} + +// Plan and validate without touching `dir`: what `--check-config` runs on a +// directory still in the old layout (a CI checkout mounted read-only). +pub fn check(dir: &Path) -> Result, String> { + let Some(plan) = plan(dir)? else { + return Ok(None); + }; + let scratch = std::env::temp_dir().join(format!("unified-api-migrate-{}", unique_suffix())); + let result = validate_in(dir, &plan, &scratch); + let _ = fs::remove_dir_all(&scratch); + result.map(|()| Some(plan)) +} + +// Convert `dir` in place, when it is in the old layout. Nothing is moved +// unless the converted configuration loads; a failure part way through the +// moves puts back what had been moved. +pub fn migrate_in_place(dir: &Path) -> Result, String> { + let Some(plan) = plan(dir)? else { + return Ok(None); + }; + let stamp = timestamp(); + let staging = dir.join(format!(".migrate-staging-{stamp}")); + if let Err(e) = validate_in(dir, &plan, &staging) { + let _ = fs::remove_dir_all(&staging); + return Err(e); + } + + let backup_dir = dir.join(format!(".legacy-config-{stamp}")); + let result = commit(dir, &plan, &staging, &backup_dir); + let _ = fs::remove_dir_all(&staging); + result.map(|()| Some(MigrationOutcome { plan, backup_dir })) +} + +// Where startup should load the configuration from, after converting it if +// it had to be. +#[derive(Debug)] +pub enum StartupConfig { + // Already in the item-file layout: load it where it is. + AsIs, + // Converted in place (see migrate_in_place). + Migrated(MigrationOutcome), + // The directory is read-only — a Kubernetes ConfigMap, a `:ro` bind mount + // — so the conversion was written to a private temporary copy instead, + // and THAT is what to load. Nothing is lost by it (the source of truth is + // wherever the read-only files come from, and it still needs converting), + // and the alternative was a pod that does not start. + Converted { + copy: ConvertedCopy, + plan: MigrationPlan, + }, +} + +// The temporary copy a read-only directory was converted into. It is what the +// process runs on, so it lives as long as the process: whoever holds this +// keeps the copy, and dropping it removes it. A copy is locked while in use; +// a process that never dropped its own (killed, crashed) leaves one unlocked, +// and the next start removes it — without that, a crash-looping pod on a +// persistent /tmp (an emptyDir survives container restarts) added one folder +// per restart, forever. +#[derive(Debug)] +pub struct ConvertedCopy { + dir: PathBuf, + // Held, never read: the lock lasts as long as the file stays open. + _lock: fs::File, +} + +impl ConvertedCopy { + pub fn path(&self) -> &Path { + &self.dir + } +} + +// `Drop` is Rust's destructor: it runs when the value goes out of scope, which +// for main's copy is when main returns after the graceful shutdown. +impl Drop for ConvertedCopy { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} + +const COPY_PREFIX: &str = "unified-api-config-"; +const COPY_LOCK: &str = ".in-use"; +// A copy older than this with no lock file at all is a leftover from a +// version that did not lock its copies (or of a start that died before it +// could): never a copy being created right now, which locks within +// microseconds of existing. +const UNLOCKED_GRACE: std::time::Duration = std::time::Duration::from_secs(60); + +// Remove the converted copies in `temp` that no running process holds. +fn sweep_stale_copies(temp: &Path) { + let Ok(entries) = fs::read_dir(temp) else { + return; + }; + for entry in entries.flatten() { + if !entry.file_name().to_string_lossy().starts_with(COPY_PREFIX) { + continue; + } + let path = entry.path(); + // Not through a symlink: /tmp is shared, and a link planted under + // this name must not point the removal anywhere else. + let Ok(meta) = fs::symlink_metadata(&path) else { + continue; + }; + if !meta.is_dir() { + continue; + } + let stale = match fs::File::open(path.join(COPY_LOCK)) { + // Lockable means nobody holds it. The file closes (and the lock + // goes) before the folder is removed. + Ok(lock) => lock.try_lock().is_ok(), + Err(_) => meta + .modified() + .ok() + .and_then(|m| m.elapsed().ok()) + .is_some_and(|age| age > UNLOCKED_GRACE), + }; + if stale { + let _ = fs::remove_dir_all(&path); + } + } +} + +// A new, locked, empty folder for a converted copy. +fn new_copy(temp: &Path) -> Result { + let dir = temp.join(format!("{COPY_PREFIX}{}", unique_suffix())); + fs::create_dir(&dir).map_err(|e| format!("create {}: {e}", dir.display()))?; + let lock = fs::File::create(dir.join(COPY_LOCK)) + .and_then(|file| file.lock().map(|()| file)) + .map_err(|e| { + let _ = fs::remove_dir_all(&dir); + format!("lock {}: {e}", dir.display()) + })?; + Ok(ConvertedCopy { dir, _lock: lock }) +} + +// What main runs before loading: in place when it can, in a temporary copy +// when the directory cannot be written, and an error (nothing changed) when +// the conversion would not load either way. +pub fn prepare_for_startup(dir: &Path) -> Result { + if detect(dir)?.is_empty() { + return Ok(StartupConfig::AsIs); + } + if writable(dir) { + return migrate_in_place(dir).map(|outcome| match outcome { + Some(outcome) => StartupConfig::Migrated(outcome), + None => StartupConfig::AsIs, + }); + } + let plan = plan(dir)?.ok_or("the directory stopped being in the 0.34 layout")?; + let temp = std::env::temp_dir(); + sweep_stale_copies(&temp); + // Dropped on failure, which removes it. + let copy = new_copy(&temp)?; + validate_in(dir, &plan, copy.path())?; + Ok(StartupConfig::Converted { copy, plan }) +} + +// Whether the migration could write here: a probe folder, created and +// removed. Cheaper to try than to predict (read-only filesystem, a ConfigMap, +// permissions, an overlay). +fn writable(dir: &Path) -> bool { + let probe = dir.join(format!(".migrate-probe-{}", std::process::id())); + match fs::create_dir(&probe) { + Ok(()) => { + let _ = fs::remove_dir(&probe); + true + } + Err(_) => false, + } +} + +fn commit(dir: &Path, plan: &MigrationPlan, staging: &Path, backup: &Path) -> Result<(), String> { + fs::create_dir_all(backup).map_err(|e| format!("create {}: {e}", backup.display()))?; + let mut moved_out: Vec<&String> = Vec::new(); + let mut moved_in: Vec<&String> = Vec::new(); + + let rollback = |moved_out: &[&String], moved_in: &[&String]| { + for name in moved_in { + let _ = fs::remove_file(dir.join(name)); + let mut parent = dir.join(name); + while parent.pop() && parent != dir && fs::remove_dir(&parent).is_ok() {} + } + for name in moved_out { + let _ = fs::rename(backup.join(name), dir.join(name)); + } + let _ = fs::remove_dir(backup); + }; + + for name in &plan.legacy_files { + if let Err(e) = fs::rename(dir.join(name), backup.join(name)) { + rollback(&moved_out, &moved_in); + return Err(format!("back up {name}: {e}")); + } + moved_out.push(name); + } + for name in plan.files.keys() { + let target = dir.join(name); + let step = target + .parent() + .map(fs::create_dir_all) + .unwrap_or(Ok(())) + .and_then(|()| fs::rename(staging.join(name), &target)); + if let Err(e) = step { + rollback(&moved_out, &moved_in); + return Err(format!("write {name}: {e}")); + } + moved_in.push(name); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dir_with(files: &[(&str, &str)]) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + fs::write( + dir.path().join("config.yaml"), + "server:\n host: \"127.0.0.1\"\n port: 9090\n", + ) + .unwrap(); + for (name, body) in files { + fs::write(dir.path().join(name), body).unwrap(); + } + dir + } + + const PROJECTS: &str = "prj-inv:\n name: \"Inventory\"\n git_url: \"https://example.invalid/inv.git\"\n credential_id: \"cred-gh\"\n"; + const CREDENTIALS: &str = + "cred-gh:\n name: \"GitHub\"\n type: \"token\"\n env_prefix: \"GH\"\n"; + + #[test] + fn a_new_layout_is_left_alone() { + let dir = dir_with(&[("sources.yaml", "sources: []\n")]); + assert!(detect(dir.path()).unwrap().is_empty()); + assert!(migrate_in_place(dir.path()).unwrap().is_none()); + } + + #[test] + fn a_file_in_neither_layout_is_refused() { + let dir = dir_with(&[("sources.yaml", "sources: []\nsrc-a:\n name: a\n")]); + assert!(detect(dir.path()).unwrap_err().contains("mixes")); + } + + #[test] + fn sources_sharing_a_script_share_one_connector() { + let dir = dir_with(&[ + ("projects.yaml", PROJECTS), + ("credentials.yaml", CREDENTIALS), + ( + "sources.yaml", + concat!( + "src-a:\n name: \"A\"\n connector_type: script\n project_id: prj-inv\n", + " script_path: d42/inventory.py\n script_args: [\"--list\"]\n", + " output_format: ansible\n credential_ids: [cred-gh]\n ttl_seconds: 60\n", + " timeout_seconds: 600\n config:\n tenant: a\n", + "src-b:\n name: \"B\"\n project_id: prj-inv\n", + " script_path: d42/inventory.py\n script_args: [\"--list\"]\n", + " output_format: ansible\n ttl_seconds: 60\n timeout_seconds: 600\n", + "src-c:\n name: \"C\"\n project_id: prj-inv\n", + " script_path: d42/inventory.py\n script_args: [\"--list\"]\n", + " output_format: ansible\n ttl_seconds: 60\n", + ), + ), + ]); + let plan = plan(dir.path()).unwrap().expect("legacy"); + assert_eq!( + plan.connectors["d42"], + vec!["src-a", "src-b", "src-c"], + "one connector, every user" + ); + assert_eq!( + plan.files["connectors/d42.yaml"], + "---\nconnectors:\n - name: d42\n type: \"script\"\n project: \"prj-inv\"\n \ + path: \"d42/inventory.py\"\n script_args: [\"--list\"]\n \ + output_format: \"ansible\"\n timeout_seconds: 600\n" + ); + assert_eq!( + plan.files["sources/src-a.yaml"], + "---\nsources:\n - name: src-a\n description: \"A\"\n connector: \"d42\"\n \ + credentials: [\"cred-gh\"]\n ttl_seconds: 60\n config:\n tenant: \"a\"\n" + ); + // src-c ran on the default timeout, which is not its connector's. + assert!(plan.files["sources/src-c.yaml"].contains("timeout_seconds: 300")); + assert!(plan.files["projects/prj-inv.yaml"].contains("credential: \"cred-gh\"")); + } + + #[test] + fn remote_and_static_sources_move_to_the_built_ins() { + let dir = dir_with(&[ + ("projects.yaml", PROJECTS), + ("credentials.yaml", CREDENTIALS), + ( + "sources.yaml", + concat!( + "src-edge:\n name: \"Edge\"\n connector_type: remote\n project_id: prj-inv\n", + " script_path: src-ssh-facts\n ttl_seconds: 30\n", + " config:\n url: \"https://edge.example:8443\"\n", + "src-vars:\n name: \"Vars\"\n connector_type: static_inventory\n", + " project_id: prj-inv\n script_path: static/inventory.yaml\n ttl_seconds: 60\n", + ), + ), + ]); + let plan = plan(dir.path()).unwrap().expect("legacy"); + assert!(plan.connectors.is_empty()); + let edge = &plan.files["sources/src-edge.yaml"]; + assert!(edge.contains("connector: \"remote\""), "{edge}"); + assert!(edge.contains(" source: \"src-ssh-facts\""), "{edge}"); + assert!(!edge.contains("prj-inv"), "{edge}"); + let vars = &plan.files["sources/src-vars.yaml"]; + assert!(vars.contains("connector: \"static_inventory\""), "{vars}"); + assert!(vars.contains("project: \"prj-inv\""), "{vars}"); + assert!(vars.contains("path: \"static/inventory.yaml\""), "{vars}"); + } + + #[test] + fn a_remote_script_survives_as_a_literal_block() { + let dir = dir_with(&[( + "sources.yaml", + concat!( + "src-os:\n name: \"OS\"\n connector_type: ssh\n project_id: prj-x\n", + " script_path: |-\n a=1\n printf '%s' \"$a\" | tr -d '\"\\\\'\n", + " ttl_seconds: 60\n config:\n hosts: \"a.example\"\n", + ), + )]); + let plan = plan(dir.path()).unwrap().expect("legacy"); + let file = &plan.files["connectors/ssh-script.yaml"]; + assert!( + file.contains(" path: |-\n a=1\n printf '%s' \"$a\" | tr -d '\"\\\\'\n"), + "{file}" + ); + // ...and reads back as the same text. + let value = crate::domain::yaml::to_json(file).unwrap(); + assert_eq!( + value["connectors"][0]["path"], + "a=1\nprintf '%s' \"$a\" | tr -d '\"\\\\'" + ); + assert!( + plan.notes + .iter() + .any(|n| n.contains("project_id was dropped")) + ); + } + + #[test] + fn in_place_backs_up_the_old_files_and_loads() { + let dir = dir_with(&[ + ("projects.yaml", PROJECTS), + ("credentials.yaml", CREDENTIALS), + ( + "sources.yaml", + "src-a:\n name: \"A\"\n project_id: prj-inv\n script_path: a.py\n ttl_seconds: 60\n", + ), + ("views.yaml", "---\n"), + ( + "api_keys.yaml", + "key-a:\n name: \"A\"\n env: \"KEY_A\"\n sources: [src-a]\n", + ), + ]); + let outcome = migrate_in_place(dir.path()).unwrap().expect("migrated"); + assert!(outcome.backup_dir.join("sources.yaml").is_file()); + assert!( + outcome.backup_dir.join("views.yaml").is_file(), + "an empty legacy file moves too" + ); + assert!(!dir.path().join("sources.yaml").exists()); + assert!(dir.path().join("connectors/a.yaml").is_file()); + let cfg = crate::config::load_config(dir.path().to_str().unwrap()).expect("loads"); + assert_eq!(cfg.sources["src-a"].name, "A"); + assert_eq!(cfg.sources["src-a"].connector, "a"); + assert_eq!( + cfg.projects["prj-inv"].credential_id.as_deref(), + Some("cred-gh") + ); + // Once is once: the next start finds nothing to do. + assert!(migrate_in_place(dir.path()).unwrap().is_none()); + } + + #[test] + fn a_conversion_that_would_not_load_moves_nothing() { + let original = "src-a:\n name: \"A\"\n project_id: prj-ghost\n script_path: a.py\n ttl_seconds: 60\n"; + let dir = dir_with(&[("sources.yaml", original)]); + let err = migrate_in_place(dir.path()).unwrap_err(); + assert!(err.contains("unknown project 'prj-ghost'"), "{err}"); + assert_eq!( + fs::read_to_string(dir.path().join("sources.yaml")).unwrap(), + original + ); + let leftovers: Vec<_> = fs::read_dir(dir.path()) + .unwrap() + .flatten() + .map(|e| e.file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(leftovers.len(), 2, "{leftovers:?}"); + } + + #[test] + fn check_validates_without_writing() { + let dir = dir_with(&[( + "sources.yaml", + "src-a:\n name: \"A\"\n project_id: prj-x\n script_path: /opt/a.py\n ttl_seconds: 60\n", + )]); + let err = check(dir.path()).unwrap_err(); + assert!(err.contains("unknown project 'prj-x'"), "{err}"); + assert!(dir.path().join("sources.yaml").is_file()); + } + + #[test] + fn a_read_only_directory_is_converted_into_a_copy() { + use std::os::unix::fs::PermissionsExt; + let dir = dir_with(&[ + ("projects.yaml", PROJECTS), + ("credentials.yaml", CREDENTIALS), + ( + "sources.yaml", + "src-a:\n name: \"A\"\n project_id: prj-inv\n script_path: a.py\n ttl_seconds: 60\n", + ), + ]); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o555)).unwrap(); + // root ignores permissions: nothing to observe then + if writable(dir.path()) { + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + return; + } + let prepared = prepare_for_startup(dir.path()); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let StartupConfig::Converted { copy, plan } = prepared.expect("converted") else { + panic!("expected a converted copy"); + }; + assert!( + dir.path().join("sources.yaml").is_file(), + "the original is untouched" + ); + let cfg = + crate::config::load_config(copy.path().to_str().unwrap()).expect("the copy loads"); + assert_eq!(cfg.sources["src-a"].connector, "a"); + assert_eq!(plan.connectors["a"], vec!["src-a"]); + let path = copy.path().to_path_buf(); + drop(copy); + assert!(!path.exists(), "the copy goes with its holder"); + } + + // Counting repeats of a base name gave the second `foo.py` the name + // `foo-2`, which a script `foo-2.py` had as its own: one connector + // replaced the other, and src-c ran `foo.py --x`. + #[test] + fn a_generated_name_never_takes_another_connectors() { + let dir = dir_with(&[ + ("projects.yaml", PROJECTS), + ("credentials.yaml", CREDENTIALS), + ( + "sources.yaml", + concat!( + "src-a:\n name: \"A\"\n project_id: prj-inv\n script_path: foo.py\n ttl_seconds: 60\n", + "src-b:\n name: \"B\"\n project_id: prj-inv\n script_path: foo.py\n", + " script_args: [\"--x\"]\n ttl_seconds: 60\n", + "src-c:\n name: \"C\"\n project_id: prj-inv\n script_path: foo-2.py\n ttl_seconds: 60\n", + ), + ), + ]); + let outcome = migrate_in_place(dir.path()).unwrap().expect("migrated"); + assert_eq!( + outcome.plan.connectors.len(), + 3, + "{:?}", + outcome.plan.connectors + ); + let cfg = crate::config::load_config(dir.path().to_str().unwrap()).expect("loads"); + let runs = |id: &str| { + let s = &cfg.sources[id]; + (s.script_path.clone(), s.script_args.clone()) + }; + assert_eq!(runs("src-a"), ("foo.py".to_string(), vec![])); + assert_eq!( + runs("src-b"), + ("foo.py".to_string(), vec!["--x".to_string()]) + ); + assert_eq!(runs("src-c"), ("foo-2.py".to_string(), vec![])); + assert_eq!(cfg.sources["src-c"].connector, "foo-2"); + assert_eq!(cfg.sources["src-b"].connector, "foo-3"); + } + + // A script in a folder called `remote` (or `static_inventory`) is not the + // built-in connector of that name, and must not be refused as if it were. + #[test] + fn a_script_named_like_a_built_in_gets_a_name_of_its_own() { + let dir = dir_with(&[ + ("projects.yaml", PROJECTS), + ("credentials.yaml", CREDENTIALS), + ( + "sources.yaml", + concat!( + "src-a:\n name: \"A\"\n project_id: prj-inv\n script_path: remote/inventory.py\n ttl_seconds: 60\n", + "src-b:\n name: \"B\"\n project_id: prj-inv\n script_path: static_inventory.py\n ttl_seconds: 60\n", + ), + ), + ]); + let plan = check(dir.path()) + .expect("the conversion loads") + .expect("legacy"); + let names: Vec<&String> = plan.connectors.keys().collect(); + assert_eq!(names, ["remote-2", "static_inventory-2"]); + } + + // Number-looking text is text: 0.34 handed `1.10` to the script as + // "1.10", and the converted files must say exactly what the old ones did. + #[test] + fn number_looking_values_are_written_as_they_were() { + let dir = dir_with(&[ + ("projects.yaml", PROJECTS), + ("credentials.yaml", CREDENTIALS), + ( + "sources.yaml", + concat!( + "src-a:\n name: 2024\n project_id: prj-inv\n script_path: a.py\n", + " script_args: [\"--account\", 0x1F, 7]\n timeout_seconds: +600\n ttl_seconds: 60\n", + " config:\n API_VERSION: 1.10\n ACCOUNT: 0x1F\n", + " BIG: 123456789012345678901234\n FLAG: True\n PORT: 80\n", + ), + ), + ( + "enrichers.yaml", + concat!( + "enr-a:\n name: \"E\"\n target_id: src-a\n project_id: prj-inv\n", + " script_path: e.py\n script_args: [\"--days\", 7, 1.10]\n", + ), + ), + ]); + let plan = plan(dir.path()).unwrap().expect("legacy"); + let source = &plan.files["sources/src-a.yaml"]; + for line in [ + " description: 2024\n", + " API_VERSION: 1.10\n", + " ACCOUNT: 0x1F\n", + " BIG: 123456789012345678901234\n", + " FLAG: True\n", + " PORT: 80\n", + ] { + assert!(source.contains(line), "{line:?} in:\n{source}"); + } + let connector = &plan.files["connectors/a.yaml"]; + assert!( + connector.contains("script_args: [\"--account\", \"0x1F\", \"7\"]\n"), + "{connector}" + ); + assert!(connector.contains("timeout_seconds: 600\n"), "{connector}"); + let enricher = &plan.files["enrichers/enr-a.yaml"]; + assert!( + enricher.contains("script_args: [\"--days\", 7, 1.10]\n"), + "{enricher}" + ); + + migrate_in_place(dir.path()).unwrap().expect("migrated"); + let cfg = crate::config::load_config(dir.path().to_str().unwrap()).expect("loads"); + let source = &cfg.sources["src-a"]; + assert_eq!(source.name, "2024"); + assert_eq!(source.config["API_VERSION"], "1.10"); + assert_eq!(source.config["ACCOUNT"], "0x1F"); + assert_eq!(source.config["BIG"], "123456789012345678901234"); + assert_eq!(source.config["FLAG"], "True"); + assert_eq!(source.script_args, ["--account", "0x1F", "7"]); + assert_eq!(source.timeout_seconds, 600); + assert_eq!(cfg.enrichers["enr-a"].script_args, ["--days", "7", "1.10"]); + } + + // Copies of a read-only directory: one still in use is locked and kept, + // one whose process is gone is removed, and so is an old unlocked one. + #[test] + fn stale_converted_copies_are_swept() { + let temp = tempfile::tempdir().unwrap(); + let live = new_copy(temp.path()).unwrap(); + // What a killed process leaves: the folder, its lock released. + let unlocked = temp.path().join(format!("{COPY_PREFIX}1-dead")); + fs::create_dir(&unlocked).unwrap(); + fs::File::create(unlocked.join(COPY_LOCK)).unwrap(); + let fresh = temp.path().join(format!("{COPY_PREFIX}2-starting")); + fs::create_dir(&fresh).unwrap(); + let unrelated = temp.path().join("something-else"); + fs::create_dir(&unrelated).unwrap(); + + sweep_stale_copies(temp.path()); + + assert!(live.path().is_dir(), "a copy in use stays"); + assert!(!unlocked.exists(), "an unlocked copy is removed"); + assert!(fresh.exists(), "a folder that may be mid-creation stays"); + assert!(unrelated.exists()); + } + + #[test] + fn connector_names() { + let key = |kind: &str, path: &str| ConnectorKey { + kind: kind.into(), + project: None, + script_path: path.into(), + script_args: vec![], + host_args: vec![], + output_format: None, + }; + assert_eq!( + connector_base_name(&key("script", "dynamic/d42/inventory.py")), + "d42" + ); + assert_eq!( + connector_base_name(&key("script", "dynamic/taxonomy/inventory_taxonomy_cat.sh")), + "taxonomy" + ); + assert_eq!( + connector_base_name(&key("script", "dynamic/unified-api/src-patching.py")), + "src-patching" + ); + assert_eq!( + connector_base_name(&key("ssh", "gather_facts")), + "ssh-gather-facts" + ); + assert_eq!(connector_base_name(&key("ssh", "a=1\nb")), "ssh-script"); + } +} diff --git a/src/domain/api_key.rs b/src/domain/api_key.rs index d56101c..e7823c9 100644 --- a/src/domain/api_key.rs +++ b/src/domain/api_key.rs @@ -15,6 +15,9 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(deny_unknown_fields)] pub struct ApiKeyDef { + // The display label: the item's `description`, falling back to its + // identity (see config::take). The identity is the map key — what + // references, URLs and cache keys use. pub name: String, // Environment variable that holds the secret value of this key diff --git a/src/domain/connector.rs b/src/domain/connector.rs new file mode 100644 index 0000000..217e085 --- /dev/null +++ b/src/domain/connector.rs @@ -0,0 +1,181 @@ +use serde::Deserialize; + +use super::source::OutputFormat; + +// A connector says HOW to talk to a kind of system: which script to run, from +// which project, with which arguments, and how to read what it prints. A +// source says WHICH instance of that system (its credentials, its `config`, +// its schedule and TTLs) and names its connector. Ten Device42 tenants are +// ten sources and one connector; before connectors existed the script, its +// arguments and its output format were copied into each of the ten. +// +// Two connectors are built in and need no item: `remote` (another +// unified-api, see adapters/out/connectors/remote.rs) and `static_inventory` +// (an Ansible YAML inventory read from a project checkout). Neither has +// anything a user could configure about the HOW: the protocol is fixed, and +// the WHERE (the remote URL, the file in the checkout) is per source. +// +// `name` is the display label, as on every configuration item: the file's +// `description`, falling back to the identity (see config::take). The +// identity itself is the map key. +// +// Unknown keys are config typos: fail startup naming the key instead of +// silently applying a default (the policy is explained once, in config.rs). +#[derive(Debug, Deserialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Connector { + pub name: String, + + // `type` in the file, which is a Rust keyword — hence the rename. + #[serde(rename = "type", default)] + pub kind: ConnectorKind, + + // Project whose checkout holds the script. Required for a script + // connector whose path is relative; meaningless for ssh, whose path runs + // on the remote host (refused there, like any setting that looks active + // and is not). `project` + `path` means the same thing on every item that + // has them — a file inside that project's checkout — which is why the + // script's location is `path` here, as it is on a static_inventory source. + #[serde(default)] + pub project: Option, + + // script: the file to run (relative to the project checkout, or absolute). + // ssh: the remote command for `gather_mode: script`, or a label such as + // "gather_facts" for the built-in gather modes. + pub path: String, + + // CLI arguments, e.g. ["--list"] for an Ansible dynamic inventory script. + // For ssh in `script` gather mode they are appended to the remote command. + #[serde(default)] + pub script_args: Vec, + + // CLI arguments for a sync that names hosts, REPLACING script_args for + // that run; `{target}` becomes the requested hostnames, comma-joined. + // Script connectors only: ssh builds its own command per host. + #[serde(default)] + pub host_args: Vec, + + // What the script prints (script connectors only — ssh builds its dataset + // itself). Option so that setting it on ssh can be refused rather than + // silently ignored; unset means native. + #[serde(default)] + pub output_format: Option, + + // Default bound on a sync through this connector. A source may override + // it with its own timeout_seconds (a bigger tenant of the same system). + #[serde(default = "crate::domain::default_timeout_seconds")] + pub timeout_seconds: u64, +} + +#[derive(Debug, Deserialize, Clone, Copy, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ConnectorKind { + #[default] + Script, + Ssh, +} + +impl ConnectorKind { + pub fn name(self) -> &'static str { + match self { + ConnectorKind::Script => "script", + ConnectorKind::Ssh => "ssh", + } + } +} + +// The built-in connectors, by the name a source writes after `connector:`. +pub const REMOTE: &str = "remote"; +pub const STATIC_INVENTORY: &str = "static_inventory"; +pub const BUILT_IN: [&str; 2] = [REMOTE, STATIC_INVENTORY]; + +impl Connector { + // Everything wrong with this connector on its own; references to other + // items (the project) are checked by config validation. + pub fn errors(&self, id: &str) -> Vec { + let mut errors = Vec::new(); + if BUILT_IN.contains(&id) { + errors.push(format!( + "Connector '{id}' uses the name of a built-in connector — '{}' and '{}' \ + need no item; rename this one", + REMOTE, STATIC_INVENTORY + )); + } + if self.path.trim().is_empty() { + errors.push(format!("Connector '{id}' has an empty path")); + } + match self.kind { + ConnectorKind::Script => { + if self.project.is_none() && !std::path::Path::new(&self.path).is_absolute() { + errors.push(format!( + "Connector '{id}' has a relative path but no project — name \ + the project whose checkout holds the script, or use an absolute path" + )); + } + } + ConnectorKind::Ssh => { + if self.project.is_some() { + errors.push(format!( + "Connector '{id}' is an ssh connector and sets project — its \ + path runs on the remote host, so no checkout is used" + )); + } + if !self.host_args.is_empty() { + errors.push(format!( + "Connector '{id}' sets host_args but is not a script connector" + )); + } + if self.output_format.is_some() { + errors.push(format!( + "Connector '{id}' sets output_format, which an ssh connector ignores \ + — it builds its dataset itself" + )); + } + } + } + errors + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn connector(yaml: &str) -> Connector { + serde_saphyr::from_str(yaml).expect("connector fixture") + } + + #[test] + fn a_script_connector_defaults_to_native_output_and_the_shared_timeout() { + let c = connector("name: d42\nproject: prj\npath: d42.py\n"); + assert_eq!(c.kind, ConnectorKind::Script); + assert_eq!(c.output_format, None); + assert_eq!(c.timeout_seconds, 300); + assert!(c.errors("d42").is_empty()); + } + + #[test] + fn a_relative_script_needs_a_project() { + let c = connector("name: x\npath: x.py\n"); + assert!(c.errors("x")[0].contains("no project")); + let c = connector("name: x\npath: /opt/x.py\n"); + assert!(c.errors("x").is_empty()); + } + + #[test] + fn script_only_settings_are_refused_on_ssh() { + let c = connector( + "name: s\ntype: ssh\nproject: p\npath: gather_facts\nhost_args: [\"{target}\"]\noutput_format: ansible\n", + ); + let errors = c.errors("s").join("\n"); + assert!(errors.contains("sets project"), "{errors}"); + assert!(errors.contains("host_args"), "{errors}"); + assert!(errors.contains("output_format"), "{errors}"); + } + + #[test] + fn a_built_in_name_is_refused() { + let c = connector("name: x\npath: /opt/x.py\n"); + assert!(c.errors("remote")[0].contains("built-in")); + } +} diff --git a/src/domain/credential.rs b/src/domain/credential.rs index 609e707..131c608 100644 --- a/src/domain/credential.rs +++ b/src/domain/credential.rs @@ -17,6 +17,9 @@ pub enum CredentialType { #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(deny_unknown_fields)] pub struct Credential { + // The display label: the item's `description`, falling back to its + // identity (see config::take). The identity is the map key — what + // references, URLs and cache keys use. pub name: String, #[serde(rename = "type")] diff --git a/src/domain/endpoint.rs b/src/domain/endpoint.rs index a940a73..cde0af9 100644 --- a/src/domain/endpoint.rs +++ b/src/domain/endpoint.rs @@ -6,10 +6,15 @@ use std::collections::HashMap; #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(deny_unknown_fields)] pub struct OutputEndpoint { + // The display label: the item's `description`, falling back to its + // identity (see config::take). The identity is the map key — what + // references, URLs and cache keys use. pub name: String, - // Which sources feed this endpoint - #[serde(default)] + // Which sources feed this endpoint (`sources:` in the item file — every + // reference dropped its `_id` suffix in 0.35; the Rust field kept its + // name, which is also the one the API answers with) + #[serde(default, rename = "sources")] pub source_ids: Vec, // How the datasets become the response. Exactly one of these is set @@ -29,7 +34,7 @@ pub struct OutputEndpoint { // Project whose checkout contains the script (script_path only; None = the // path is a plain filesystem path, absolute or relative to the working dir) - #[serde(default)] + #[serde(default, rename = "project")] pub project_id: Option, // Free config for the transformer (builtin or script) diff --git a/src/domain/enricher.rs b/src/domain/enricher.rs index 801eae3..224aa52 100644 --- a/src/domain/enricher.rs +++ b/src/domain/enricher.rs @@ -13,9 +13,15 @@ use std::collections::HashMap; #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(deny_unknown_fields)] pub struct Enricher { + // The display label: the item's `description`, falling back to its + // identity (see config::take). The identity is the map key — what + // references, URLs and cache keys use. pub name: String, - // The dataset being enriched — ex: "src-ssh-pq-facts" + // The dataset being enriched — ex: "src-ssh-pq-facts". References are + // written without the `_id` suffix in the item files (`target:`); the + // Rust field keeps its name because the API still answers `target_id`. + #[serde(rename = "target")] pub target_id: String, // Script that performs the enrichment (required for script-based mode) @@ -28,11 +34,11 @@ pub struct Enricher { // Project whose checkout contains the script (None = script_path is a // plain filesystem path, absolute or relative to the working directory) - #[serde(default)] + #[serde(default, rename = "project")] pub project_id: Option, // Declarative merge: the source dataset to copy fields from - #[serde(default)] + #[serde(default, rename = "source")] pub source_id: Option, // Declarative merge: which keys to take from the source, on a group's vars diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 9399674..3c131c0 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -1,5 +1,6 @@ pub mod api_key; pub mod cache_entry; +pub mod connector; pub mod credential; pub mod dataset; pub mod endpoint; diff --git a/src/domain/project.rs b/src/domain/project.rs index fed95c4..1a6ef97 100644 --- a/src/domain/project.rs +++ b/src/domain/project.rs @@ -9,6 +9,9 @@ use serde::Deserialize; #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(deny_unknown_fields)] pub struct GitProject { + // The display label: the item's `description`, falling back to its + // identity (see config::take). The identity is the map key — what + // references, URLs and cache keys use. pub name: String, pub git_url: String, @@ -16,7 +19,9 @@ pub struct GitProject { #[serde(default = "default_branch")] pub branch: String, - // Credential for private repos (GitHub token, SSH key, etc.) + // Credential for private repos (GitHub token, SSH key, etc.) — + // `credential:` in the item file + #[serde(default, rename = "credential")] pub credential_id: Option, // Seconds between periodic re-pulls (0 or None = no periodic sync). diff --git a/src/domain/source.rs b/src/domain/source.rs index bb17ba6..4a4dece 100644 --- a/src/domain/source.rs +++ b/src/domain/source.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use utoipa::ToSchema; +use super::connector::{self, Connector, ConnectorKind}; use super::dataset::Dataset; use super::sync_mode::SyncMode; @@ -11,12 +12,14 @@ pub enum ConnectorType { #[default] Script, Ssh, - // Ansible static YAML inventory read from disk (script_path = path to - // the inventory file, typically inside a git project checkout) + // Ansible static YAML inventory read from disk: the built-in + // `static_inventory` connector (the source's `path`, inside its + // `project` checkout) StaticInventory, // Federation: fetch another unified-api instance's cached dataset over - // HTTP (script_path = the source id ON THE REMOTE instance; config.url = - // the remote base URL). Ages are propagated so freshness stays truthful. + // HTTP — the built-in `remote` connector (config.url = the remote base + // URL, config.source = the source id ON THE REMOTE instance). Ages are + // propagated so freshness stays truthful. Remote, } @@ -34,13 +37,54 @@ pub enum OutputFormat { Ansible, } -// Unknown keys are config typos: fail startup naming the key instead of -// silently applying a default (the policy is explained once, in config.rs). +impl OutputFormat { + // As written after `output_format:` in a connector, for anything that + // shows the configuration back (the connectors listing). + pub fn name(self) -> &'static str { + match self { + OutputFormat::Native => "native", + OutputFormat::Ansible => "ansible", + } + } +} + +// A source as it RUNS: the per-instance settings from its item file, with +// its connector's settings resolved into it at load (see SourceDef::resolve). +// +// Resolving once, at load, is what lets everything downstream — sync, the +// scheduler, the connectors, the views — keep reading one flat struct, and it +// means a reload that changes a connector shows up as a change to every source +// using it, which is exactly the set of sources whose next sync behaves +// differently. +// +// Configuration files are NEVER read as this: their shape is SourceDef. +// `Deserialize` is here for code that builds a resolved source from a compact +// description of it — tests, mostly — which is why most fields have a default +// a fixture can leave out. (`deny_unknown_fields` still applies: a fixture +// with a typo fails like a file would.) #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(deny_unknown_fields)] pub struct Source { + // The display label: the item's description, or its identity. pub name: String, - pub project_id: String, + + // The connector this source names (`connector:` in its file) — an item + // name, or one of the built-ins in domain::connector::BUILT_IN. + #[serde(default)] + pub connector: String, + + // ---- resolved from the connector ------------------------------------- + #[serde(default)] + pub connector_type: ConnectorType, + + // The checkout a relative script_path (or static inventory path) resolves + // in. None for remote and ssh, which run nothing from a checkout. + #[serde(default)] + pub project_id: Option, + + // What the connector adapter is handed: the script for script, the + // remote command for ssh (both from the connector's `path`), the inventory file for static_inventory and the + // source id ON THE REMOTE instance for remote. pub script_path: String, // CLI arguments passed to the script, e.g. ["--list"] for scripts that @@ -66,30 +110,31 @@ pub struct Source { #[serde(default)] pub host_args: Vec, - // Whether a host-scoped sync also takes the vars of the groups it places - // its hosts in. Off by default: a granular call answers about a HOST, and - // letting it redefine what a group means for every other member is a bigger - // statement than the sync was asked to make. Worth turning on for a script - // whose granular mode deliberately describes those groups in full. - #[serde(default)] - pub host_sync_updates_group_vars: bool, - - #[serde(default)] - pub connector_type: ConnectorType, - // Format of the script's stdout (script connector only) #[serde(default)] pub output_format: OutputFormat, // Maximum seconds a sync may run before it is aborted — protects the - // scheduler and API from a hung connector script (default 300) + // scheduler and API from a hung connector script. The source's own + // timeout_seconds when it sets one, else its connector's (default 300). #[serde(default = "crate::domain::default_timeout_seconds")] pub timeout_seconds: u64, + // ---- the source's own settings --------------------------------------- + + // Whether a host-scoped sync also takes the vars of the groups it places + // its hosts in. Off by default: a granular call answers about a HOST, and + // letting it redefine what a group means for every other member is a bigger + // statement than the sync was asked to make. Worth turning on for a script + // whose granular mode deliberately describes those groups in full. + #[serde(default)] + pub host_sync_updates_group_vars: bool, + #[serde(default)] pub sync_mode: SyncMode, - // Vec = list of credential IDs that the connector needs + // Vec = list of credential ids that the connector needs + // (`credentials:` in the file) #[serde(default)] pub credential_ids: Vec, @@ -100,6 +145,7 @@ pub struct Source { // sources get no startup jitter (the times were chosen deliberately) but // back off on failure exactly like interval sources, by letting // occurrences pass. + #[serde(default)] pub schedule: Option, // Automatic sync interval in seconds (simple alternative to cron) @@ -147,6 +193,220 @@ pub struct Source { pub config: HashMap, } +// A source as written in its item file (`sources:`). Only what is particular +// to THIS instance of a system: which connector, which credentials, which +// `config`, when to sync and how long data stays fresh. How to talk to the +// system lives on the connector it names. +// +// `name` is the display label (the file's `description`, falling back to the +// identity — see config::take); the identity is the map key. +// +// Unknown keys are config typos: fail startup naming the key instead of +// silently applying a default (the policy is explained once, in config.rs). +// That is also what turns a 0.34 source pasted into a 0.35 file into an error +// naming `connector_type` or `script_path`, rather than a source that half +// works. +#[derive(Debug, Deserialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SourceDef { + pub name: String, + + // A connector item's name, or a built-in: `remote`, `static_inventory`. + pub connector: String, + + #[serde(default)] + pub credentials: Vec, + + // static_inventory only: the project whose checkout holds the inventory, + // and the file inside it. Every other connector carries its own project. + #[serde(default)] + pub project: Option, + + #[serde(default)] + pub path: Option, + + // Overrides the connector's timeout for this source only. + #[serde(default)] + pub timeout_seconds: Option, + + #[serde(default)] + pub host_sync_updates_group_vars: bool, + + #[serde(default)] + pub sync_mode: SyncMode, + + #[serde(default)] + pub schedule: Option, + + #[serde(default)] + pub sync_interval_seconds: Option, + + pub ttl_seconds: u64, + + #[serde(default)] + pub ttl_overrides: TtlOverrides, + + #[serde(default)] + pub hosts_from_source: Option, + + #[serde(default)] + pub allow_on_demand_refresh: bool, + + #[serde(default)] + pub advertise_scope: Option, + + #[serde(default)] + pub config: HashMap, +} + +impl SourceDef { + // The runtime source: this item plus its connector. Every problem that + // can be known from the two of them is returned, so one load names all + // of them; references to other items (projects, credentials, sources) + // are checked afterwards by config validation, against the resolved set. + pub fn resolve( + &self, + id: &str, + connectors: &HashMap, + ) -> Result> { + let mut errors = Vec::new(); + let default_timeout = crate::domain::default_timeout_seconds(); + + // (connector_type, project, script_path, script_args, host_args, + // output_format, connector timeout) + let resolved = match self.connector.as_str() { + connector::REMOTE => { + for key in ["url", "source"] { + if !self.config.contains_key(key) { + errors.push(format!( + "Source '{id}' uses the remote connector but has no '{key}' in \ + config — set config.url to the other instance and config.source \ + to the source id there" + )); + } + } + self.refuse_location(id, "remote", &mut errors); + let remote_id = self.config.get("source").cloned().unwrap_or_default(); + Some(( + ConnectorType::Remote, + None, + remote_id, + Vec::new(), + Vec::new(), + OutputFormat::Native, + default_timeout, + )) + } + connector::STATIC_INVENTORY => { + if self.project.is_none() { + errors.push(format!( + "Source '{id}' uses the static_inventory connector but sets no \ + project — name the project whose checkout holds the inventory" + )); + } + let path = match &self.path { + Some(path) if !path.trim().is_empty() => path.clone(), + _ => { + errors.push(format!( + "Source '{id}' uses the static_inventory connector but sets no \ + path — the inventory file inside the project checkout" + )); + String::new() + } + }; + Some(( + ConnectorType::StaticInventory, + self.project.clone(), + path, + Vec::new(), + Vec::new(), + OutputFormat::Native, + default_timeout, + )) + } + name => match connectors.get(name) { + None => { + errors.push(format!( + "Source '{id}' references unknown connector '{name}'" + )); + None + } + Some(c) => { + self.refuse_location(id, name, &mut errors); + let kind = match c.kind { + ConnectorKind::Script => ConnectorType::Script, + ConnectorKind::Ssh => ConnectorType::Ssh, + }; + Some(( + kind, + c.project.clone(), + c.path.clone(), + c.script_args.clone(), + c.host_args.clone(), + c.output_format.unwrap_or_default(), + c.timeout_seconds, + )) + } + }, + }; + + let Some(( + connector_type, + project_id, + script_path, + script_args, + host_args, + output_format, + connector_timeout, + )) = resolved + else { + return Err(errors); + }; + if !errors.is_empty() { + return Err(errors); + } + + Ok(Source { + name: self.name.clone(), + connector: self.connector.clone(), + connector_type, + project_id, + script_path, + script_args, + host_args, + output_format, + timeout_seconds: self.timeout_seconds.unwrap_or(connector_timeout), + host_sync_updates_group_vars: self.host_sync_updates_group_vars, + sync_mode: self.sync_mode.clone(), + credential_ids: self.credentials.clone(), + schedule: self.schedule.clone(), + sync_interval_seconds: self.sync_interval_seconds, + ttl_seconds: self.ttl_seconds, + ttl_overrides: self.ttl_overrides.clone(), + hosts_from_source: self.hosts_from_source.clone(), + allow_on_demand_refresh: self.allow_on_demand_refresh, + advertise_scope: self.advertise_scope.clone(), + config: self.config.clone(), + }) + } + + // `project` and `path` are the static_inventory connector's: on any other + // source they would look like they select a script, and they do not. + fn refuse_location(&self, id: &str, connector: &str, errors: &mut Vec) { + for (key, set) in [ + ("project", self.project.is_some()), + ("path", self.path.is_some()), + ] { + if set { + errors.push(format!( + "Source '{id}' sets {key}, which only a static_inventory source uses — \ + the '{connector}' connector decides what runs" + )); + } + } + } +} + // An explicit ownership claim for scope advertising — the same group/host // vocabulary a view's Ownership uses, minus the resolving source (that is the // CONSUMER's business: the central decides which of its inventories the @@ -455,6 +715,108 @@ mod tests { serde_saphyr::from_str(yaml).unwrap() } + // A source item resolved against two connectors: `facts` (ssh) and `inv` + // (a script). `body` is the rest of the item, after its connector. + fn resolved(connector: &str, body: &str) -> Result> { + let connectors: HashMap = [ + ("facts", "name: facts\ntype: ssh\npath: gather_facts\n"), + ( + "inv", + "name: inv\nproject: prj\npath: inv.py\nscript_args: [\"--list\"]\noutput_format: ansible\ntimeout_seconds: 120\n", + ), + ] + .into_iter() + .map(|(id, yaml)| (id.to_string(), serde_saphyr::from_str(yaml).unwrap())) + .collect(); + let def: SourceDef = serde_saphyr::from_str(&format!( + "name: s\nconnector: {connector}\nttl_seconds: 60\n{body}" + )) + .expect("source fixture"); + def.resolve("src-s", &connectors) + } + + fn source(connector: &str, body: &str) -> Source { + resolved(connector, body).expect("resolves") + } + + #[test] + fn a_source_takes_how_from_its_connector_and_which_from_itself() { + let s = source("inv", "credentials: [cred-a]\nconfig:\n tenant: a\n"); + assert_eq!(s.connector, "inv"); + assert_eq!(s.connector_type, ConnectorType::Script); + assert_eq!(s.project_id.as_deref(), Some("prj")); + assert_eq!(s.script_path, "inv.py"); + assert_eq!(s.script_args, vec!["--list"]); + assert_eq!(s.output_format, OutputFormat::Ansible); + assert_eq!(s.timeout_seconds, 120, "the connector's timeout"); + assert_eq!(s.credential_ids, vec!["cred-a"]); + assert_eq!(s.config["tenant"], "a"); + + let s = source("inv", "timeout_seconds: 900\n"); + assert_eq!( + s.timeout_seconds, 900, + "a source may override its connector" + ); + } + + #[test] + fn the_built_in_connectors_take_their_location_from_the_source() { + let s = source( + "remote", + "config:\n url: \"https://edge.example\"\n source: src-ssh-facts\n", + ); + assert_eq!(s.connector_type, ConnectorType::Remote); + assert_eq!(s.script_path, "src-ssh-facts"); + assert_eq!(s.project_id, None); + + let s = source( + "static_inventory", + "project: prj\npath: static/inventory.yaml\n", + ); + assert_eq!(s.connector_type, ConnectorType::StaticInventory); + assert_eq!(s.script_path, "static/inventory.yaml"); + assert_eq!(s.project_id.as_deref(), Some("prj")); + } + + #[test] + fn what_a_connector_needs_from_its_source_is_named() { + let errors = resolved("remote", "").unwrap_err().join("\n"); + assert!( + errors.contains("no 'url'") && errors.contains("no 'source'"), + "{errors}" + ); + + let errors = resolved("static_inventory", "").unwrap_err().join("\n"); + assert!( + errors.contains("no project") && errors.contains("no path"), + "{errors}" + ); + + let errors = resolved("inv", "project: prj\npath: x.yaml\n") + .unwrap_err() + .join("\n"); + assert!( + errors.contains("sets project") && errors.contains("sets path"), + "{errors}" + ); + + let errors = resolved("ghost", "").unwrap_err().join("\n"); + assert!( + errors.contains("Source 'src-s' references unknown connector 'ghost'"), + "{errors}" + ); + } + + #[test] + fn a_0_34_source_fails_naming_the_keys_that_moved() { + let err = serde_saphyr::from_str::( + "name: s\nconnector_type: script\nproject_id: p\nscript_path: x\nttl_seconds: 60\n", + ) + .expect_err("legacy keys are unknown keys") + .to_string(); + assert!(err.contains("connector_type"), "{err}"); + } + #[test] fn empty_pattern_selects_every_host() { let (specs, warnings) = hfs("source: src-a\n").resolve(&dataset()); @@ -544,13 +906,13 @@ mod tests { #[test] fn an_explicit_advertise_scope_beats_the_derived_pattern() { - let source: Source = serde_saphyr::from_str(concat!( - "name: s\nproject_id: p\nscript_path: x\nttl_seconds: 60\n", - "connector_type: \"ssh\"\n", - "hosts_from_source:\n source: \"src-inv\"\n match_pattern:\n groups: [\"derived\"]\n", - "advertise_scope:\n groups: [\"explicit\"]\n", - )) - .expect("source fixture"); + let source = source( + "facts", + concat!( + "hosts_from_source:\n source: \"src-inv\"\n match_pattern:\n groups: [\"derived\"]\n", + "advertise_scope:\n groups: [\"explicit\"]\n", + ), + ); let claim = source.advertised_scope().expect("declared"); assert_eq!(claim.groups, vec!["explicit"]); @@ -559,13 +921,13 @@ mod tests { #[test] fn a_hosts_from_source_pattern_is_the_advertised_scope() { - let source: Source = serde_saphyr::from_str(concat!( - "name: s\nproject_id: p\nscript_path: x\nttl_seconds: 60\n", - "connector_type: \"ssh\"\n", - "hosts_from_source:\n source: \"src-inv\"\n match_pattern:\n", - " groups: [\"datacenter_dc2\"]\n hosts: [\"appliance.dc2.example\"]\n", - )) - .expect("source fixture"); + let source = source( + "facts", + concat!( + "hosts_from_source:\n source: \"src-inv\"\n match_pattern:\n", + " groups: [\"datacenter_dc2\"]\n hosts: [\"appliance.dc2.example\"]\n", + ), + ); let claim = source.advertised_scope().expect("declared"); assert_eq!(claim.groups, vec!["datacenter_dc2"]); @@ -575,12 +937,7 @@ mod tests { #[test] fn an_empty_pattern_advertises_a_spelled_out_catch_all() { - let source: Source = serde_saphyr::from_str(concat!( - "name: s\nproject_id: p\nscript_path: x\nttl_seconds: 60\n", - "connector_type: \"ssh\"\n", - "hosts_from_source:\n source: \"src-inv\"\n", - )) - .expect("source fixture"); + let source = source("facts", "hosts_from_source:\n source: \"src-inv\"\n"); let claim = source.advertised_scope().expect("declared"); assert!(claim.groups.is_empty() && claim.hosts.is_empty()); @@ -592,9 +949,7 @@ mod tests { #[test] fn an_ordinary_source_makes_no_claim() { - let source: Source = - serde_saphyr::from_str("name: s\nproject_id: p\nscript_path: x\nttl_seconds: 60\n") - .expect("source fixture"); + let source = source("inv", ""); assert!(source.advertised_scope().is_none()); } } diff --git a/src/domain/topology.rs b/src/domain/topology.rs index 4f2bf21..a7fb67c 100644 --- a/src/domain/topology.rs +++ b/src/domain/topology.rs @@ -13,6 +13,9 @@ use crate::domain::source::{ConnectorType, Source}; // peer that asked for it. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] pub struct Topology { + /// What the instance is called (its `server.name`), when it says + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, /// The running version, as the instance reports it pub version: String, /// Sources configured on the instance. Absent when the peer is too old to @@ -24,6 +27,54 @@ pub struct Topology { pub peers: Vec, } +// Why a peer was not asked, as a caller outside this instance's admins reads +// it: the resolver's own message names env vars and Vault paths. +const NOT_ASKED: &str = "not asked: this instance cannot authenticate to it"; +// A link's last failure, as an instance this one pushes its map to reads it. +const PULL_FAILED: &str = "the last pull failed — its error is on the instance that pulls"; + +impl Topology { + // What a caller who is not an admin here may see of the peers. They were + // asked with THIS instance's key, which may be an admin key over there: + // their own peers, URLs and counts are that key's view, not the caller's, + // so only each peer's name and version are kept. And why a peer was not + // asked stays generic (see NOT_ASKED); /credentials keeps that for + // admins. + pub fn redacted(mut self) -> Topology { + for peer in &mut self.peers { + if let Some(instance) = peer.instance.as_mut() { + instance.sources = None; + instance.peers = Vec::new(); + } + if peer + .error + .as_deref() + .is_some_and(|e| e.starts_with("not asked:") && !e.contains("depth limit")) + { + peer.error = Some(NOT_ASKED.to_string()); + } + } + self + } + + // What this instance pushes to its peers (PUT /topology/federation). The + // peer's admins are not this instance's: they get the redacted map, and a + // failing link says it fails, not why — a sync error can name this + // instance's env vars and Vault paths as well. Every peer gets the same + // map, its own link included, so siblings still show where it sits. + pub fn for_push(self) -> Topology { + let mut topology = self.redacted(); + for peer in &mut topology.peers { + for source in &mut peer.sources { + if source.last_error.is_some() { + source.last_error = Some(PULL_FAILED.to_string()); + } + } + } + topology + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] pub struct Peer { /// The peer's base URL, as configured on the sources that read from it @@ -56,10 +107,66 @@ pub struct PeerSource { pub age_seconds: Option, /// Failed pulls since the last success — 0 while healthy pub consecutive_failures: u32, + /// Seconds since the last pull that worked: when data last moved along + /// this link. Absent until one has. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_pull_age_seconds: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub last_error: Option, } +// What an instance that reads from this one sends it: the federation as THAT +// instance sees it, and the address it reaches this one at. An edge cannot +// map the federation itself — it reads from nobody, and does not know who +// reads from it — so the instance above tells it, over the same link and +// with the same key its remote sources already use. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +pub struct FederationPush { + /// This instance's address, as the sender's remote sources spell it + pub url: String, + /// The sender's own address (its `server.url`), when it declares one + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sender_url: Option, + /// The sender's own facts + #[serde(default)] + pub sender: InstanceSummary, + /// The sender's topology, this instance included as one of its peers + pub topology: Topology, +} + +// What the sender says about itself, for the card the edge draws it as: the +// same facts its own Topology page shows on its own card. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, ToSchema)] +pub struct InstanceSummary { + /// Its source ids, sorted (views left out: they hold no data of their own) + pub sources: Vec, + /// Hosts across those sources' caches, added up + pub hosts: usize, + /// Groups across those sources' caches, added up + pub groups: usize, + pub endpoints: usize, + /// Age of its oldest cached data; null when nothing is cached + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oldest_age_seconds: Option, +} + +// A pushed map as GET /topology/federation serves it back. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +pub struct Federation { + /// The API key the sender authenticated with: which instance it was + pub pushed_by: String, + /// This instance's address, as the sender spells it + pub url: String, + /// The sender's address, when its `server.url` declares one + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sender_url: Option, + pub sender: InstanceSummary, + pub topology: Topology, + /// Seconds since it arrived. Senders push every minute, so an old one + /// means the sender stopped (or lost its way here). + pub age_seconds: u64, +} + // The peers of a configuration: every remote source the caller may see, // grouped by the URL it reads from (trailing slashes ignored, so two spellings // of one instance are one peer). Sorted by URL, then source id, so the answer @@ -127,6 +234,77 @@ mod tests { serde_saphyr::from_str(&yaml).unwrap() } + fn peer(instance: Option, error: Option<&str>) -> Peer { + Peer { + url: "https://edge.example".to_string(), + sources: Vec::new(), + instance: instance.map(Box::new), + error: error.map(str::to_string), + } + } + + #[test] + fn a_redacted_map_keeps_each_peers_version_and_no_secret_locations() { + let behind = Topology { + name: None, + version: "0.33.0".to_string(), + sources: Some(12), + peers: vec![peer(None, None)], + }; + let topology = Topology { + name: None, + version: "0.34.0".to_string(), + sources: Some(1), + peers: vec![ + peer(Some(behind), None), + peer(None, Some("not asked: Env var 'EDGE_TOKEN' not set")), + peer(None, Some("not asked: depth limit reached")), + ], + }; + let seen = topology.redacted(); + let instance = seen.peers[0].instance.as_ref().unwrap(); + assert_eq!(instance.version, "0.33.0"); + assert!(instance.peers.is_empty() && instance.sources.is_none()); + assert!( + !seen.peers[1] + .error + .as_deref() + .unwrap() + .contains("EDGE_TOKEN") + ); + assert_eq!( + seen.peers[2].error.as_deref(), + Some("not asked: depth limit reached") + ); + } + + #[test] + fn a_pushed_map_says_a_link_fails_but_not_why() { + let mut link = peer(None, None); + link.sources.push(PeerSource { + source_id: "src-dc1".to_string(), + remote_source: "src-fleet".to_string(), + cached: false, + hosts: None, + groups: None, + age_seconds: None, + consecutive_failures: 3, + last_pull_age_seconds: None, + last_error: Some("Vault path 'kv/central/dc1' denied".to_string()), + }); + let topology = Topology { + name: None, + version: "0.35.0".to_string(), + sources: Some(1), + peers: vec![link], + }; + let pushed = topology.for_push(); + let source = &pushed.peers[0].sources[0]; + assert_eq!(source.consecutive_failures, 3); + let error = source.last_error.as_deref().unwrap(); + assert!(!error.contains("kv/central"), "{error}"); + } + #[test] fn credentials_in_a_remote_url_are_never_shown() { assert_eq!( diff --git a/src/domain/view.rs b/src/domain/view.rs index e697202..2eec22f 100644 --- a/src/domain/view.rs +++ b/src/domain/view.rs @@ -23,6 +23,9 @@ use super::dataset::Dataset; #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(deny_unknown_fields)] pub struct View { + // The display label: the item's `description`, falling back to its + // identity (see config::take). The identity is the map key — what + // references, URLs and cache keys use. pub name: String, // Declared order is significant: the FIRST member that claims a host wins diff --git a/src/domain/yaml.rs b/src/domain/yaml.rs index cb1c4f3..266ac40 100644 --- a/src/domain/yaml.rs +++ b/src/domain/yaml.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use serde::Deserializer; +use serde::de::{DeserializeOwned, DeserializeSeed, MapAccess, SeqAccess, Visitor}; use serde_json::{Map, Number, Value}; use serde_saphyr::granit_parser::{Event, Options, Parser, ScalarStyle, StrInput, Tag}; @@ -33,6 +35,18 @@ const MIN_ALIAS_BUDGET: usize = 1_000_000; const CORE_TAG: &str = "tag:yaml.org,2002:"; pub fn to_json(doc: &str) -> Result { + read(doc, false) +} + +// The same tree as `to_json`, except that a plain scalar resolving to a number +// or a boolean stays a string of its source text: `1.10` is "1.10", `0x1F` is +// "0x1F". The text a configuration file wrote, for the fields where text is +// what is meant — see `from_json_with_text`. +pub fn to_text_json(doc: &str) -> Result { + read(doc, true) +} + +fn read(doc: &str, verbatim: bool) -> Result { // Comments are not data; without this they arrive as events of their own. let mut options = Options::default(); options.emit_comments = false; @@ -41,12 +55,15 @@ pub fn to_json(doc: &str) -> Result { anchors: HashMap::new(), events: 0, copied: 0, + verbatim, }; reader.stream() } struct Reader<'a> { parser: Parser<'a, StrInput<'a>>, + // Numbers and booleans kept as their text (`to_text_json`). + verbatim: bool, // anchor id → (value, its node count, which is what copying it costs) anchors: HashMap, events: usize, @@ -111,7 +128,13 @@ impl<'a> Reader<'a> { .to_string(), ); } - (scalar(&text, style, tag.as_deref()), 1, anchor) + let value = match scalar(&text, style, tag.as_deref()) { + Value::Number(_) | Value::Bool(_) if self.verbatim => { + Value::String(text.to_string()) + } + value => value, + }; + (value, 1, anchor) } Event::SequenceStart(_, anchor, _) => { let mut items = Vec::new(); @@ -254,6 +277,187 @@ fn float(text: &str) -> Option { unsigned.parse::().ok().filter(|f| f.is_finite()) } +// Types `typed` (a `to_json` tree) as `T`, the way the typed YAML reader +// before 0.35 did: a field that wants TEXT takes a plain scalar's source text +// from `text` (the `to_text_json` tree of the same document), whatever the +// scalar looks like. `script_args: ["--days", 7]` is `["--days", "7"]`, and a +// `config:` value written `1.10` reaches its script as "1.10" — through +// numbers it would be 1.1, and `0x1F` would be 31. Every other field reads +// `typed` exactly as `serde_json::from_value` would. +pub fn from_json_with_text(typed: Value, text: Value) -> Result { + T::deserialize(Pair { typed, text }).map_err(|e| e.to_string()) +} + +// A node of each tree, walked together. Both trees come from one document, so +// they have the same shape; a node missing from `text` reads as Null, which +// only means "no text known" (the typed value is used). +struct Pair { + typed: Value, + text: Value, +} + +// A trait is like an interface in Java: implementing serde's `Deserializer` +// for `Pair` is what lets any `#[derive(Deserialize)]` struct read from it. +// Most methods are left to `deserialize_any` by the macro below; only the +// ones whose answer depends on the field's wanted type are written out. +impl<'de> Deserializer<'de> for Pair { + type Error = serde_json::Error; + + fn deserialize_any>(self, visitor: V) -> Result { + match (self.typed, self.text) { + (Value::Array(typed), text) => { + let text = match text { + Value::Array(text) => text, + _ => Vec::new(), + }; + let len = typed.len(); + let mut seq = Seq { + typed: typed.into_iter(), + text: text.into_iter(), + }; + let value = visitor.visit_seq(&mut seq)?; + if seq.typed.len() == 0 { + Ok(value) + } else { + Err(serde::de::Error::invalid_length( + len, + &"fewer elements in array", + )) + } + } + (Value::Object(typed), text) => { + let text = match text { + Value::Object(text) => text, + _ => Map::new(), + }; + let len = typed.len(); + let mut map = Entries { + typed: typed.into_iter(), + text, + value: None, + }; + let value = visitor.visit_map(&mut map)?; + if map.typed.len() == 0 { + Ok(value) + } else { + Err(serde::de::Error::invalid_length( + len, + &"fewer elements in map", + )) + } + } + (typed, _) => typed.deserialize_any(visitor), + } + } + + // The point of the pair: text wanted, a number or boolean found, its + // source text given. + fn deserialize_str>(self, visitor: V) -> Result { + self.deserialize_string(visitor) + } + + fn deserialize_string>(self, visitor: V) -> Result { + match (self.typed, self.text) { + (Value::Number(_) | Value::Bool(_), Value::String(text)) => visitor.visit_string(text), + (typed, _) => typed.deserialize_string(visitor), + } + } + + fn deserialize_option>(self, visitor: V) -> Result { + match self.typed { + Value::Null => visitor.visit_none(), + _ => visitor.visit_some(self), + } + } + + fn deserialize_newtype_struct>( + self, + _name: &'static str, + visitor: V, + ) -> Result { + visitor.visit_newtype_struct(self) + } + + // Enums in configuration are words (`sync_mode: merge`): the typed value + // reads them as it always has. + fn deserialize_enum>( + self, + name: &'static str, + variants: &'static [&'static str], + visitor: V, + ) -> Result { + self.typed.deserialize_enum(name, variants, visitor) + } + + serde::forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char bytes byte_buf + unit unit_struct seq tuple tuple_struct map struct identifier ignored_any + } +} + +struct Seq { + typed: std::vec::IntoIter, + text: std::vec::IntoIter, +} + +impl<'de> SeqAccess<'de> for Seq { + type Error = serde_json::Error; + + fn next_element_seed>( + &mut self, + seed: S, + ) -> Result, Self::Error> { + let Some(typed) = self.typed.next() else { + return Ok(None); + }; + let text = self.text.next().unwrap_or(Value::Null); + seed.deserialize(Pair { typed, text }).map(Some) + } + + fn size_hint(&self) -> Option { + Some(self.typed.len()) + } +} + +struct Entries { + typed: serde_json::map::IntoIter, + text: Map, + // The value whose key was just handed out, with its text. + value: Option, +} + +impl<'de> MapAccess<'de> for Entries { + type Error = serde_json::Error; + + fn next_key_seed>( + &mut self, + seed: S, + ) -> Result, Self::Error> { + let Some((key, typed)) = self.typed.next() else { + return Ok(None); + }; + let text = self.text.remove(&key).unwrap_or(Value::Null); + self.value = Some(Pair { typed, text }); + // Keys are text already (see the reader): a plain string key. + seed.deserialize(Value::String(key)).map(Some) + } + + fn next_value_seed>( + &mut self, + seed: S, + ) -> Result { + let pair = self + .value + .take() + .ok_or_else(|| serde::de::Error::custom("a map value was read before its key"))?; + seed.deserialize(pair) + } + + fn size_hint(&self) -> Option { + Some(self.typed.len()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index 9c50b4c..884daaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,7 @@ use ports::secrets::SecretsPort; // and build() finalizes the construction. pub struct AppBuilder { sources: HashMap, + connectors: HashMap, views: HashMap, enrichers: HashMap, endpoints: HashMap, @@ -64,12 +65,15 @@ pub struct AppBuilder { refresh_max_concurrent: usize, shutdown_grace_seconds: u64, max_body_bytes: usize, + url: Option, + name: Option, } impl AppBuilder { pub fn new() -> Self { Self { sources: HashMap::new(), + connectors: HashMap::new(), views: HashMap::new(), enrichers: HashMap::new(), endpoints: HashMap::new(), @@ -99,6 +103,8 @@ impl AppBuilder { shutdown_grace_seconds: 20, // axum's own default, mirrored like the three above max_body_bytes: 2 * 1024 * 1024, + url: None, + name: None, } } @@ -114,6 +120,13 @@ impl AppBuilder { self } + // The connector items, for the /connectors listing. Sources built in code + // already carry what their connector says; this is only what is shown. + pub fn connectors(mut self, connectors: HashMap) -> Self { + self.connectors = connectors; + self + } + pub fn views(mut self, views: HashMap) -> Self { self.views = views; self @@ -176,6 +189,7 @@ impl AppBuilder { // boot is not a cost worth a partially-moved struct. pub fn from_config(mut self, cfg: &config::AppConfig) -> Self { self.sources = cfg.sources.clone(); + self.connectors = cfg.connectors.clone(); self.views = cfg.views.clone(); self.enrichers = cfg.enrichers.clone(); self.endpoints = cfg.endpoints.clone(); @@ -191,6 +205,14 @@ impl AppBuilder { self.refresh_max_concurrent = cfg.server.refresh_max_concurrent; self.shutdown_grace_seconds = cfg.server.shutdown_grace_seconds; self.max_body_bytes = cfg.server.max_body_bytes; + // As a reload reads it (application::config): one spelling, no + // credentials, so boot and reload agree. + self.url = cfg + .server + .url + .as_deref() + .map(|url| crate::domain::topology::without_userinfo(url.trim_end_matches('/'))); + self.name = cfg.server.name.clone(); // What this process is built from, kept whether or not the config // API is on: GET /settings shows the restart-only values in force. self.live_settings = Some(config::RestartOnlySettings::from_config(cfg)); @@ -214,6 +236,7 @@ impl AppBuilder { pub fn api_key(mut self, api_key: Option) -> Self { if let Some(secret) = api_key { self.api_keys.push(ResolvedApiKey { + id: "default".to_string(), name: "default".to_string(), secret, permissions: Permissions::Admin, @@ -295,6 +318,7 @@ impl AppBuilder { secrets_swap: reloadable, config: std::sync::RwLock::new(Arc::new(RuntimeConfig { sources: self.sources, + connectors: self.connectors, credentials: self.credentials, views: self.views, enrichers: self.enrichers, @@ -310,6 +334,8 @@ impl AppBuilder { refresh_max_concurrent: self.refresh_max_concurrent, ldap: self.ui.ldap.clone(), ui_home: self.ui.home().to_string(), + url: self.url, + name: self.name, })), config_store: self.config_store, live_settings: std::sync::RwLock::new(self.live_settings), @@ -320,6 +346,7 @@ impl AppBuilder { sessions: std::sync::Arc::new(crate::adapters::r#in::http::session::SessionStore::new()), directory: self.directory, peers: self.peers, + federation: Arc::new(crate::application::topology::FederationStore::new()), sync_health: Arc::new(domain::sync_health::SyncHealthRegistry::new()), advertised_scopes: Arc::new(domain::source::AdvertisedScopeRegistry::new()), enrich_health: Arc::new(domain::sync_health::SyncHealthRegistry::new()), diff --git a/src/main.rs b/src/main.rs index 114dc67..2403bf5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,21 +4,76 @@ use unified_api::adapters::out::git::cli::CliGit; #[tokio::main] async fn main() { + let args: Vec = std::env::args().skip(1).collect(); + + // Convert a 0.34 configuration directory into the 0.35 layout and exit: + // what startup does on its own, offered as a command for a configuration + // repository to run once and commit. Same code path, so a repository and + // a pod upgraded in place end up with the same files. + if let Some(i) = args.iter().position(|arg| arg == "--migrate-config") { + let dir = args + .get(i + 1) + .cloned() + .or_else(|| std::env::var("CONFIG_DIR").ok()) + .unwrap_or_else(|| "config".to_string()); + match unified_api::config::migrate::migrate_in_place(std::path::Path::new(&dir)) { + Ok(Some(outcome)) => { + for line in outcome.plan.summary() { + println!("{line}"); + } + println!( + "migrated {dir}; the old files are in {}", + outcome.backup_dir.display() + ); + return; + } + Ok(None) => { + println!("{dir} is already in the item-file layout; nothing to do"); + return; + } + Err(e) => { + eprintln!("migration of {dir} FAILED, nothing was changed: {e}"); + std::process::exit(1); + } + } + } + // Validate-only mode for CI on config repositories: load and validate the // configuration exactly as startup would, report, and exit — nothing // binds, no scheduler starts, and no secrets are resolved (a CI runner // validating YAML has no reason to hold the deployment's env vars). // Plain prints rather than tracing: this output is for a human or a CI // log, not for a log pipeline. - if std::env::args().skip(1).any(|arg| arg == "--check-config") { + // + // A directory still in the 0.34 layout is converted in a scratch copy + // and THAT is checked: the directory may be mounted read-only, and the + // question CI asks is whether this version will start on it — which, + // since startup migrates, it will exactly when the conversion loads. + if args.iter().any(|arg| arg == "--check-config") { let config_dir = std::env::var("CONFIG_DIR").unwrap_or_else(|_| "config".to_string()); + match unified_api::config::migrate::check(std::path::Path::new(&config_dir)) { + Ok(Some(plan)) => { + println!( + "configuration OK ({config_dir}) in the 0.34 layout: startup would \ + convert it ({}); run --migrate-config to do it now", + plan.summary().join("; ") + ); + return; + } + Ok(None) => {} + Err(e) => { + eprintln!("configuration INVALID ({config_dir}): {e}"); + std::process::exit(1); + } + } match unified_api::config::load_config(&config_dir) { Ok(cfg) => { println!( - "configuration OK ({}): {} sources, {} views, {} credentials, \ - {} enrichers, {} endpoints, {} projects", + "configuration OK ({}): {} sources, {} connectors, {} views, \ + {} credentials, {} enrichers, {} endpoints, {} projects", config_dir, cfg.sources.len(), + cfg.connectors.len(), cfg.views.len(), cfg.credentials.len(), cfg.enrichers.len(), @@ -45,6 +100,60 @@ async fn main() { .init(); let config_dir = std::env::var("CONFIG_DIR").unwrap_or_else(|_| "config".to_string()); + + // A pod upgraded from 0.34 still has the old files on its volume. + // Convert them once, loudly, before loading — in place, or into a private + // copy when the directory is read-only (a ConfigMap) — or refuse to start, + // having changed nothing, when the conversion would not load. + // Held to the end of main: it is the directory this process runs on, and + // dropping it removes it (see migrate::ConvertedCopy). + let mut _converted_copy = None; + let config_dir = match unified_api::config::migrate::prepare_for_startup(std::path::Path::new( + &config_dir, + )) { + Ok(unified_api::config::migrate::StartupConfig::AsIs) => config_dir, + Ok(unified_api::config::migrate::StartupConfig::Migrated(outcome)) => { + warn!( + dir = %config_dir, + backup = %outcome.backup_dir.display(), + "Configuration directory was in the 0.34 layout and has been migrated \ + to item files; the old files are kept in the backup folder. Commit \ + the new layout wherever this directory comes from, or it will be \ + pushed back over it" + ); + for line in outcome.plan.summary() { + warn!(dir = %config_dir, "Migration: {}", line); + } + config_dir + } + Ok(unified_api::config::migrate::StartupConfig::Converted { copy, plan }) => { + warn!( + dir = %config_dir, + copy = %copy.path().display(), + "Configuration directory is in the 0.34 layout and read-only: it was \ + converted into a temporary copy, which is what this process runs. \ + Convert the source of these files (unified-api --migrate-config) — \ + every restart converts them again, and writes through the config \ + API land in the copy" + ); + for line in plan.summary() { + warn!(dir = %config_dir, "Migration: {}", line); + } + let dir = copy.path().to_string_lossy().into_owned(); + _converted_copy = Some(copy); + dir + } + Err(e) => { + error!( + dir = %config_dir, + "Refusing to start: the configuration is in the 0.34 layout and could \ + not be migrated (nothing was changed): {}", + e + ); + std::process::exit(1); + } + }; + let cfg = match unified_api::config::load_config(&config_dir) { Ok(cfg) => cfg, Err(e) => { @@ -83,6 +192,7 @@ async fn main() { info!( sources = cfg.sources.len(), + connectors = cfg.connectors.len(), views = cfg.views.len(), credentials = cfg.credentials.len(), enrichers = cfg.enrichers.len(), diff --git a/src/ports/git.rs b/src/ports/git.rs index 4dc5e16..9d3cbad 100644 --- a/src/ports/git.rs +++ b/src/ports/git.rs @@ -8,6 +8,7 @@ use crate::domain::project::GitProject; // Boxed future alias, same pattern as the other ports (see connector.rs // for why Pin>). pub type GitFuture<'a> = Pin> + Send + 'a>>; +pub type RevisionFuture<'a> = Pin> + Send + 'a>>; // GitPort — keep a local checkout of a project's repository up to date. // The concrete implementation shells out to the git binary; the trait exists @@ -23,6 +24,13 @@ pub trait GitPort: Send + Sync { project: &GitProject, credentials: &HashMap, ) -> GitFuture<'_>; + + // The commit the checkout in `dir` is at (full id), for the projects page: + // "which scripts are running" is answered by a commit. None when it cannot + // tell (no checkout yet); the default lets test doubles leave it out. + fn revision(&self, _dir: &Path) -> RevisionFuture<'_> { + Box::pin(async { None }) + } } #[derive(Debug)] diff --git a/src/ports/peer.rs b/src/ports/peer.rs index 5430255..0e3ae23 100644 --- a/src/ports/peer.rs +++ b/src/ports/peer.rs @@ -1,16 +1,27 @@ use std::future::Future; use std::pin::Pin; -use crate::domain::topology::Topology; +use crate::domain::topology::{FederationPush, Topology}; // PeerPort — ask another unified-api instance to describe itself, for the // federation map (GET /api/v1/topology). Read-only, and never on the data // path: a peer that cannot be asked is shown as such, nothing fails. pub trait PeerPort: Send + Sync { fn describe(&self, request: PeerRequest) -> PeerFuture; + // Tell the peer what the federation looks like from here, so its own + // map can show where it sits (PUT /api/v1/topology/federation). + fn push(&self, request: PeerRequest, body: FederationPush) -> PushFuture; } pub type PeerFuture = Pin> + Send>>; +pub type PushFuture = Pin> + Send>>; + +#[derive(Debug, PartialEq)] +pub enum PushError { + // The peer predates the route: nothing wrong, it just cannot show it. + Unsupported, + Failed(String), +} // Owned, so the requests to every peer can run concurrently on their own tasks. pub struct PeerRequest { diff --git a/src/state.rs b/src/state.rs index 689583b..5923f49 100644 --- a/src/state.rs +++ b/src/state.rs @@ -22,7 +22,13 @@ use crate::ports; // half of an old configuration and half of a new one), and the next request // picks up the new one. pub struct RuntimeConfig { + // Each source with its connector already folded in (SourceDef::resolve): + // what a sync reads. pub sources: HashMap, + // The connector items, as written. Read only to describe them (the + // /connectors listing) — a sync never looks here, its source already + // carries everything the connector said. + pub connectors: HashMap, // Carried but never resolved from here — the secrets chain holds its own // copy. It lives in the snapshot so a reload can say WHICH credential // changed, which is the first question asked when a sync starts failing @@ -69,6 +75,12 @@ pub struct RuntimeConfig { pub ldap: Option, // `server.ui.home`: where a sign-in with no page to return to lands. pub ui_home: String, + // `server.url`: this instance's address, sent with the federation map. + // Read per push, so it reloads. + pub url: Option, + // `server.name`: what the instance is called on the Topology page, in + // its own answer and in the map it pushes. Read per request. + pub name: Option, } // Bumped on every reload, and watched by the scheduler: the tasks of the @@ -168,6 +180,9 @@ pub struct AppState { // Asks other unified-api instances to describe themselves, for the // federation map. Read-only and never on the data path. pub peers: Arc, + // The federation maps instances reading from this one pushed here, so an + // edge can show where it sits (it maps nothing itself). + pub federation: Arc, // The browsable HTML API's settings. Static like `projects_dir`, and for // the same reason: it decides which middlewares and routes are in the @@ -280,17 +295,25 @@ impl AppState { // None = the id is not in sources.yaml, same contract as sources.get(). pub fn source_for_sync(&self, id: &str) -> Option { let mut source = self.config().sources.get(id)?.clone(); - if !matches!(source.connector_type, ConnectorType::Ssh) { + // Only script and static_inventory sources read a file on this + // machine; ssh runs its script_path remotely and remote speaks HTTP, + // and neither carries a project. + if let Some(project_id) = source.project_id.clone() + && !matches!( + source.connector_type, + ConnectorType::Ssh | ConnectorType::Remote + ) + { source.script_path = crate::application::scripts::resolve_script_path( &self.projects_dir, id, - &source.project_id, + &project_id, &source.script_path, ); // Same per-execution stance for the project's virtualenv: if one // exists on disk, the process adapter puts it on the child's PATH if let Some(bin) = - crate::application::scripts::venv_bin_dir(&self.projects_dir, &source.project_id) + crate::application::scripts::venv_bin_dir(&self.projects_dir, &project_id) { source .config diff --git a/tests/api_integration_test.rs b/tests/api_integration_test.rs index 91efb06..3137a2b 100644 --- a/tests/api_integration_test.rs +++ b/tests/api_integration_test.rs @@ -109,13 +109,14 @@ fn app_with_duplicate_group_members() -> axum::Router { fn script_source(script_path: &str) -> unified_api::domain::source::Source { unified_api::domain::source::Source { name: "Test Source".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: script_path.to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: unified_api::domain::source::ConnectorType::Script, sync_mode: unified_api::domain::sync_mode::SyncMode::Replace, credential_ids: vec![], @@ -235,6 +236,27 @@ async fn list_source_groups_returns_names_and_counts() { assert!(groups[0].get("hosts").is_none()); } +// One group: its variables and size, without the rest of the dataset. +#[tokio::test] +async fn get_source_group_returns_its_vars() { + let (status, body) = get( + app_with_demo_data(), + "/api/v1/sources/src-demo/groups/section9", + ) + .await; + assert_eq!(status, StatusCode::OK); + let group: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(group["name"], "section9"); + assert_eq!(group["host_count"], 1); + assert_eq!(group["vars"]["ntp_server"], "ntp.section9.net"); + assert!(group.get("hosts").is_none()); + + // A group the source does not have: 404, saying which. + let (status, body) = get(app_with_demo_data(), "/api/v1/sources/src-demo/groups/nope").await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(body.contains("no group 'nope'"), "{body}"); +} + #[tokio::test] async fn list_source_hosts_returns_sorted_names_only() { let (status, body) = get(app_with_demo_data(), "/api/v1/sources/src-demo/hosts").await; @@ -333,13 +355,14 @@ async fn readyz_returns_503_before_sync() { "src-test".to_string(), unified_api::domain::source::Source { name: "Test".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/inventory.py".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: unified_api::domain::source::ConnectorType::Script, sync_mode: unified_api::domain::sync_mode::SyncMode::Replace, credential_ids: vec![], @@ -372,13 +395,14 @@ async fn readyz_require_all_sources_waits_for_every_source() { id.to_string(), unified_api::domain::source::Source { name: "Test".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/inventory.py".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: unified_api::domain::source::ConnectorType::Script, sync_mode: unified_api::domain::sync_mode::SyncMode::Replace, credential_ids: vec![], @@ -910,11 +934,13 @@ async fn get_with_key(app: axum::Router, path: &str, key: &str) -> (StatusCode, fn app_with_scoped_keys() -> axum::Router { let keys = vec![ ResolvedApiKey { + id: "admin".to_string(), name: "admin".to_string(), secret: "admin-secret".to_string(), permissions: Permissions::Admin, }, ResolvedApiKey { + id: "alpha-only".to_string(), name: "alpha-only".to_string(), secret: "alpha-secret".to_string(), permissions: Permissions::Scoped { @@ -1505,6 +1531,7 @@ async fn endpoint_errors_carry_a_body_like_every_other_route() { let app = unified_api::AppBuilder::new() .endpoints(endpoints) .api_keys(vec![ResolvedApiKey { + id: "scoped".to_string(), name: "scoped".to_string(), secret: "scoped-secret".to_string(), permissions: Permissions::Scoped { @@ -1535,6 +1562,7 @@ async fn an_unknown_endpoint_id_says_so() { let app = unified_api::AppBuilder::new() .endpoints(HashMap::new()) .api_keys(vec![ResolvedApiKey { + id: "admin".to_string(), name: "admin".to_string(), secret: "admin-secret".to_string(), permissions: Permissions::Admin, @@ -1566,7 +1594,7 @@ async fn a_script_appearing_in_a_checkout_after_boot_is_used_by_the_next_sync() let projects_dir = tempfile::tempdir().unwrap(); let mut source = script_source("fetch.py"); - source.project_id = "prj-test".to_string(); + source.project_id = Some("prj-test".to_string()); let mut sources = std::collections::HashMap::new(); sources.insert("src-checkout".to_string(), source); @@ -1705,7 +1733,7 @@ async fn responses_carry_a_request_id() { // ========================================================================= fn app_with_declarative_enricher() -> (axum::Router, std::sync::Arc) { let enricher: unified_api::domain::enricher::Enricher = serde_saphyr::from_str( - "name: Copy infinibox\ntarget_id: src-demo\nsource_id: src-storage\nfields: [\"infinibox\"]\n", + "name: Copy infinibox\ntarget: src-demo\nsource: src-storage\nfields: [\"infinibox\"]\n", ) .expect("enricher fixture"); let mut enrichers = std::collections::HashMap::new(); @@ -1804,7 +1832,7 @@ async fn running_an_unknown_enricher_is_a_404_naming_it() { #[tokio::test] async fn metrics_exposes_task_health_gauges() { let enricher: unified_api::domain::enricher::Enricher = - serde_saphyr::from_str("name: E\ntarget_id: src-demo\nscript_path: e.py\n").unwrap(); + serde_saphyr::from_str("name: E\ntarget: src-demo\nscript_path: e.py\n").unwrap(); let mut enrichers = std::collections::HashMap::new(); enrichers.insert("en-gauges".to_string(), enricher); let project: unified_api::domain::project::GitProject = @@ -1890,7 +1918,7 @@ async fn an_enrichers_trigger_reaches_the_script_inside_source_config() { ); let enricher: unified_api::domain::enricher::Enricher = serde_saphyr::from_str(concat!( "name: Probe\n", - "target_id: src-probe\n", + "target: src-probe\n", "script_path: \"tests/adapters/out/enrichers/env_probe.py\"\n", )) .expect("enricher fixture"); @@ -2024,7 +2052,7 @@ fn app_for_group_var_enrichment() -> (axum::Router, std::sync::Arc( - "name: vars\ntarget_id: src-target\nsource_id: src-vars\nfields: [\"cmdb_role\"]\n", + "name: vars\ntarget: src-target\nsource: src-vars\nfields: [\"cmdb_role\"]\n", ) .expect("enricher fixture"), )]); diff --git a/tests/audit_log_test.rs b/tests/audit_log_test.rs index 73787ae..eb455ab 100644 --- a/tests/audit_log_test.rs +++ b/tests/audit_log_test.rs @@ -24,6 +24,7 @@ impl std::io::Write for CapturedLog { fn admin_key() -> Vec { vec![ResolvedApiKey { + id: "key-ops".to_string(), name: "key-ops".to_string(), secret: "sekrit".to_string(), permissions: Permissions::Admin, @@ -172,7 +173,7 @@ fn config_app(key_env: &str) -> (axum::Router, tempfile::TempDir) { std::fs::write( dir.path().join("api_keys.yaml"), format!( - "key-audit:\n name: \"auditor\"\n env: \"{}\"\n role: \"admin\"\n", + "api_keys:\n - name: key-audit\n description: \"auditor\"\n env: \"{}\"\n role: \"admin\"\n", key_env ), ) diff --git a/tests/check_config_test.rs b/tests/check_config_test.rs index 86c9218..3b1dd53 100644 --- a/tests/check_config_test.rs +++ b/tests/check_config_test.rs @@ -65,7 +65,7 @@ fn a_broken_cross_reference_fails_like_startup_would() { // A source naming a project that does not exist — valid YAML, invalid config std::fs::write( dir.path().join("sources.yaml"), - "src-a:\n name: \"A\"\n project_id: \"prj-missing\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", + "sources:\n - name: src-a\n description: \"A\"\n connector: c-src-a\n ttl_seconds: 60\nconnectors:\n - name: c-src-a\n project: \"prj-missing\"\n path: \"x.py\"\n", ) .unwrap(); diff --git a/tests/config_api_test.rs b/tests/config_api_test.rs index a86be0e..c202a39 100644 --- a/tests/config_api_test.rs +++ b/tests/config_api_test.rs @@ -44,7 +44,7 @@ fn config_dir(key_env: &str) -> tempfile::TempDir { fn api_keys_yaml(key_env: &str) -> String { format!( - "key-pipeline:\n name: \"pipeline\"\n env: \"{}\"\n role: \"admin\"\n", + "api_keys:\n - name: key-pipeline\n description: \"pipeline\"\n env: \"{}\"\n role: \"admin\"\n", key_env ) } @@ -149,9 +149,10 @@ fn json(body: &str) -> serde_json::Value { } // A source that references a project declared in the same push. -const PROJECTS: &str = - "prj-inv:\n name: \"Inventory\"\n git_url: \"https://example.invalid/i.git\"\n"; -const SOURCES: &str = "src-dc4:\n name: \"DC4\"\n project_id: \"prj-inv\"\n script_path: \"inv.py\"\n ttl_seconds: 300\n"; +const PROJECTS: &str = "projects:\n - name: prj-inv\n description: \"Inventory\"\n git_url: \"https://example.invalid/i.git\"\n"; +// A source and the connector it names, in one file: a file may hold several +// sections. +const SOURCES: &str = "connectors:\n - name: inv\n project: \"prj-inv\"\n path: \"inv.py\"\nsources:\n - name: src-dc4\n description: \"DC4\"\n connector: inv\n ttl_seconds: 300\n"; // ========================================================================= // It is off unless it is turned on @@ -186,6 +187,7 @@ async fn a_restricted_key_cannot_read_the_configuration() { let (app, _) = unified_api::AppBuilder::new() .from_config(&cfg) .api_keys(vec![ResolvedApiKey { + id: "consumer".to_string(), name: "consumer".to_string(), secret: KEY.to_string(), permissions: Permissions::Scoped { @@ -226,16 +228,12 @@ async fn the_inventory_names_what_is_there_and_what_is_not() { .collect(); assert_eq!( names, - vec!["config.yaml", "projects.yaml", "api_keys.yaml"], - "listed in the order the loader reads them" - ); - assert!( - inventory["missing"] - .as_array() - .expect("missing") - .iter() - .any(|n| n == "sources.yaml") + vec!["api_keys.yaml", "config.yaml", "projects.yaml"], + "listed by path, the order the loader reads them" ); + // Only config.yaml is ever required; every other file is an item file + // that is there or not. + assert_eq!(inventory["missing"], serde_json::json!([])); assert_eq!(inventory["valid"], true); assert_eq!(inventory["reload_pending"], false); assert_eq!(inventory["generation"], 0); @@ -272,13 +270,62 @@ async fn a_file_is_served_verbatim_and_revalidates_with_its_etag() { } #[tokio::test] -async fn a_name_the_loader_does_not_read_is_a_404_naming_the_ones_it_does() { +async fn a_path_the_loader_would_never_read_is_a_404_naming_the_rule() { let dir = config_dir("UNIFIED_API_TEST_KEY_UNKNOWN"); let (app, _) = app_at(dir.path()); - let (status, body) = send(&app, get("/api/v1/config/secrets.yaml")).await; + for bad in ["secrets.json", ".hidden.yaml", "a/.git/x.yaml"] { + let (status, body) = send(&app, get(&format!("/api/v1/config/{bad}"))).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{bad}"); + assert!(body.contains("not a configuration path"), "body: {}", body); + } + // A valid path that is simply not there yet. + let (status, body) = send(&app, get("/api/v1/config/sources/x.yaml")).await; assert_eq!(status, StatusCode::NOT_FOUND); - assert!(body.contains("credentials.yaml"), "body: {}", body); + assert!(body.contains("not present"), "body: {}", body); +} + +// A file in a folder is read, written and deleted by its path, slashes +// included, and the folders a delete leaves empty go with it. +#[tokio::test] +async fn a_file_in_a_folder_is_addressed_by_its_path() { + let dir = config_dir("UNIFIED_API_TEST_KEY_NESTED"); + write(dir.path(), "projects.yaml", PROJECTS); + let (app, state) = app_at(dir.path()); + + let (status, body) = send( + &app, + put_yaml( + "/api/v1/config/sources/dc4/src-dc4.yaml?reload=true", + SOURCES, + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + read(dir.path(), "sources/dc4/src-dc4.yaml").as_deref(), + Some(SOURCES) + ); + assert!(state.config().sources.contains_key("src-dc4")); + assert!(state.config().connectors.contains_key("inv")); + + let (status, body) = send(&app, get("/api/v1/config/sources/dc4/src-dc4.yaml")).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, SOURCES); + + let delete = Request::builder() + .method("DELETE") + .uri("/api/v1/config/sources/dc4/src-dc4.yaml?reload=true") + .header("x-api-key", KEY) + .body(axum::body::Body::empty()) + .expect("request"); + let (status, body) = send(&app, delete).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!( + !dir.path().join("sources").exists(), + "the emptied folders go too" + ); + assert!(state.config().sources.is_empty()); } #[tokio::test] @@ -306,8 +353,8 @@ async fn validation_reports_every_problem_at_once_and_writes_nothing() { let body = serde_json::json!({ "files": { - "sources.yaml": "src-a:\n name: \"A\"\n project_id: \"prj-ghost\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", - "enrichers.yaml": "enr-a:\n name: \"A\"\n target_id: \"src-nowhere\"\n script_path: \"e.py\"\n" + "sources.yaml": "sources:\n - name: src-a\n description: \"A\"\n connector: c-src-a\n ttl_seconds: 60\nconnectors:\n - name: c-src-a\n project: \"prj-ghost\"\n path: \"x.py\"\n", + "enrichers.yaml": "enrichers:\n - name: enr-a\n description: \"A\"\n target: \"src-nowhere\"\n script_path: \"e.py\"\n" } }); let request = Request::builder() @@ -344,7 +391,7 @@ async fn a_write_that_would_not_load_is_refused_and_the_directory_is_untouched() &app, put_yaml( "/api/v1/config/sources.yaml", - "src-a:\n name: \"A\"\n project_id: \"prj-ghost\"\n script_path: \"x.py\"\n ttl_seconds: 60\n", + "connectors:\n - name: c-a\n project: \"prj-ghost\"\n path: \"x.py\"\nsources:\n - name: src-a\n connector: c-a\n ttl_seconds: 60\n", ), ) .await; @@ -569,8 +616,7 @@ async fn a_key_whose_env_var_is_missing_fails_the_whole_operation() { let dir = config_dir("UNIFIED_API_TEST_KEY_ABSENT"); let (app, _) = app_at(dir.path()); - let keys = - "key-new:\n name: \"new\"\n env: \"UNIFIED_API_TEST_ABSENT_VAR\"\n role: \"admin\"\n"; + let keys = "api_keys:\n - name: key-new\n description: \"new\"\n env: \"UNIFIED_API_TEST_ABSENT_VAR\"\n role: \"admin\"\n"; let (status, body) = send( &app, put_yaml("/api/v1/config/api_keys.yaml?reload=true", keys), @@ -597,7 +643,7 @@ async fn a_reloaded_key_file_is_in_force_on_the_next_request() { // SAFETY: a name unique to this test, so no other test observes it. unsafe { std::env::set_var("UNIFIED_API_TEST_ROTATED_KEY", "rotated-secret") }; - let keys = "key-rotated:\n name: \"rotated\"\n env: \"UNIFIED_API_TEST_ROTATED_KEY\"\n role: \"admin\"\n"; + let keys = "api_keys:\n - name: key-rotated\n description: \"rotated\"\n env: \"UNIFIED_API_TEST_ROTATED_KEY\"\n role: \"admin\"\n"; let (status, body) = send( &app, diff --git a/tests/connectors_test.rs b/tests/connectors_test.rs new file mode 100644 index 0000000..eacdebf --- /dev/null +++ b/tests/connectors_test.rs @@ -0,0 +1,235 @@ +// The connectors routes through the real router, on a configuration loaded +// from item files exactly as main loads it: what each connector runs, which +// sources use it, and how those sources are doing. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use http_body_util::BodyExt; +use tower::ServiceExt; + +use unified_api::adapters::r#in::http::auth::{Permissions, ResolvedApiKey}; +use unified_api::config::load_config; + +const SCRIPT: &str = "tests/adapters/out/connectors/inventory.py"; + +fn config_dir() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + let write = |name: &str, body: String| { + let path = dir.path().join(name); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, body).unwrap(); + }; + write( + "config.yaml", + "server:\n host: \"127.0.0.1\"\n port: 9090\n".to_string(), + ); + write( + "projects/prj-inv.yaml", + "projects:\n - name: prj-inv\n git_url: \"https://example.invalid/inv.git\"\n" + .to_string(), + ); + let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(SCRIPT); + write( + "connectors/sample.yaml", + format!( + "connectors:\n - name: sample\n description: \"Sample inventory\"\n \ + project: prj-inv\n path: \"{}\"\n script_args: [\"--list\"]\n \ + timeout_seconds: 120\n", + script.display() + ), + ); + write( + "sources/a/src-ok.yaml", + "sources:\n - name: src-ok\n description: \"Works\"\n connector: sample\n \ + ttl_seconds: 60\n config:\n scenario: default\n" + .to_string(), + ); + write( + "sources/b/src-broken.yaml", + "sources:\n - name: src-broken\n connector: sample\n ttl_seconds: 60\n \ + timeout_seconds: 30\n config:\n scenario: error\n" + .to_string(), + ); + write( + "sources/src-edge.yaml", + "sources:\n - name: src-edge\n connector: remote\n ttl_seconds: 60\n \ + config:\n url: \"http://127.0.0.1:9\"\n source: src-facts\n" + .to_string(), + ); + dir +} + +fn app(admin: bool) -> axum::Router { + let dir = config_dir(); + let cfg = load_config(dir.path().to_str().unwrap()).expect("fixture loads"); + let permissions = if admin { + Permissions::Admin + } else { + Permissions::Scoped { + sources: ["src-ok".to_string()].into_iter().collect(), + endpoints: Default::default(), + } + }; + unified_api::AppBuilder::new() + .from_config(&cfg) + .api_keys(vec![ResolvedApiKey { + id: "test".to_string(), + name: "test".to_string(), + secret: "k".to_string(), + permissions, + }]) + .build() +} + +async fn call(app: &axum::Router, method: &str, uri: &str) -> (StatusCode, serde_json::Value) { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("x-api-key", "k") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + ( + status, + serde_json::from_slice(&body).unwrap_or(serde_json::Value::Null), + ) +} + +#[tokio::test] +async fn the_list_names_what_each_connector_runs_and_who_uses_it() { + let app = app(true); + let (status, body) = call(&app, "GET", "/api/v1/connectors").await; + assert_eq!(status, StatusCode::OK); + let list = body.as_array().unwrap(); + let ids: Vec<&str> = list + .iter() + .map(|c| c["connector_id"].as_str().unwrap()) + .collect(); + assert_eq!( + ids, + vec!["sample", "remote", "static_inventory"], + "configured first, the built-ins last" + ); + + let sample = &list[0]; + assert_eq!(sample["name"], "Sample inventory"); + assert_eq!(sample["connector_type"], "script"); + assert_eq!(sample["built_in"], false); + assert_eq!(sample["project_id"], "prj-inv"); + assert!(sample["path"].as_str().unwrap().ends_with("inventory.py")); + assert_eq!(sample["script_args"], serde_json::json!(["--list"])); + assert_eq!(sample["output_format"], "native"); + assert_eq!(sample["timeout_seconds"], 120); + let users: Vec<(&str, u64)> = sample["used_by"] + .as_array() + .unwrap() + .iter() + .map(|u| { + ( + u["source_id"].as_str().unwrap(), + u["timeout_seconds"].as_u64().unwrap(), + ) + }) + .collect(); + assert_eq!( + users, + vec![("src-broken", 30), ("src-ok", 120)], + "each source with the timeout it actually syncs under" + ); + assert_eq!( + sample["health"], + serde_json::json!({"sources": 2, "healthy": 0, "failing": 0, "never_synced": 2}) + ); + + let remote = &list[1]; + assert_eq!(remote["built_in"], true); + assert_eq!(remote["connector_type"], "remote"); + assert!(remote["path"].is_null()); + assert!(remote.get("script_path").is_none()); + assert_eq!(remote["used_by"][0]["source_id"], "src-edge"); + assert_eq!(list[2]["used_by"], serde_json::json!([])); +} + +#[tokio::test] +async fn health_counts_follow_the_syncs_of_the_sources_using_it() { + let app = app(true); + call(&app, "POST", "/api/v1/sources/src-ok/sync").await; + call(&app, "POST", "/api/v1/sources/src-broken/sync").await; + + let (status, body) = call(&app, "GET", "/api/v1/connectors/sample").await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + body["health"], + serde_json::json!({"sources": 2, "healthy": 1, "failing": 1, "never_synced": 0}) + ); + let broken = body["used_by"] + .as_array() + .unwrap() + .iter() + .find(|u| u["source_id"] == "src-broken") + .unwrap(); + assert_eq!(broken["failing"], true); + assert_eq!(broken["synced"], false); +} + +#[tokio::test] +async fn one_connector_by_id_built_ins_included_and_a_404_otherwise() { + let app = app(true); + let (status, body) = call(&app, "GET", "/api/v1/connectors/static_inventory").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["built_in"], true); + + let (status, body) = call(&app, "GET", "/api/v1/connectors/ghost").await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(body["error"].as_str().unwrap().contains("ghost")); +} + +#[tokio::test] +async fn a_restricted_key_cannot_read_connectors() { + let app = app(false); + for uri in ["/api/v1/connectors", "/api/v1/connectors/sample"] { + let (status, body) = call(&app, "GET", uri).await; + assert_eq!(status, StatusCode::FORBIDDEN, "{uri}"); + assert!(body["error"].is_string(), "an error carries a body"); + } + let (_, index) = call(&app, "GET", "/api/v1/").await; + assert!( + index.get("connectors").is_none(), + "no link it cannot follow" + ); +} + +#[tokio::test] +async fn a_source_and_a_project_name_their_connector() { + let app = app(true); + let (_, index) = call(&app, "GET", "/api/v1/").await; + assert_eq!(index["connectors"], "/api/v1/connectors"); + + let (_, source) = call(&app, "GET", "/api/v1/sources/src-ok").await; + assert_eq!(source["name"], "Works", "the label is the description"); + assert_eq!(source["gathering"]["connector"], "sample"); + assert_eq!(source["gathering"]["connector_type"], "script"); + assert_eq!(source["gathering"]["project_id"], "prj-inv"); + assert_eq!(source["gathering"]["timeout_seconds"], 120); + + let (_, edge) = call(&app, "GET", "/api/v1/sources/src-edge").await; + assert_eq!(edge["gathering"]["connector"], "remote"); + assert!(edge["gathering"]["project_id"].is_null()); + + let (_, projects) = call(&app, "GET", "/api/v1/projects").await; + assert_eq!( + projects[0]["used_by"]["connectors"], + serde_json::json!(["sample"]) + ); + assert_eq!( + projects[0]["used_by"]["sources"], + serde_json::json!(["src-broken", "src-ok"]) + ); +} diff --git a/tests/credentials_test.rs b/tests/credentials_test.rs index bcc2c8d..7acc72f 100644 --- a/tests/credentials_test.rs +++ b/tests/credentials_test.rs @@ -49,11 +49,13 @@ fn app() -> axum::Router { .credentials(credentials, Default::default()) .api_keys(vec![ ResolvedApiKey { + id: "admin".to_string(), name: "admin".to_string(), secret: "adm".to_string(), permissions: Permissions::Admin, }, ResolvedApiKey { + id: "awx".to_string(), name: "awx".to_string(), secret: "awx".to_string(), permissions: Permissions::Scoped { diff --git a/tests/federation_test.rs b/tests/federation_test.rs new file mode 100644 index 0000000..73d4596 --- /dev/null +++ b/tests/federation_test.rs @@ -0,0 +1,171 @@ +// The federation map pushed down to an edge: the real HTTP adapter sending +// it, the edge's route storing it, and the edge serving it back. + +use unified_api::adapters::out::peers::http::HttpPeers; +use unified_api::domain::topology::{FederationPush, Topology}; +use unified_api::ports::peer::{PeerPort, PeerRequest}; + +async fn spawn_edge() -> String { + let app = unified_api::AppBuilder::new().build(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{addr}") +} + +fn request(url: &str) -> PeerRequest { + PeerRequest { + url: url.to_string(), + api_key: None, + insecure_tls: false, + timeout_seconds: 5, + depth: 0, + } +} + +#[tokio::test] +async fn a_pushed_map_is_served_back_by_the_edge() { + let edge = spawn_edge().await; + let central = Topology { + name: None, + version: "0.35.0".to_string(), + sources: Some(30), + peers: Vec::new(), + }; + HttpPeers::new() + .push( + request(&edge), + FederationPush { + url: edge.clone(), + sender_url: Some("https://central.example".to_string()), + sender: Default::default(), + topology: central.clone(), + }, + ) + .await + .expect("the edge accepts the push"); + + let served: serde_json::Value = reqwest::get(format!("{edge}/api/v1/topology/federation")) + .await + .unwrap() + .json() + .await + .unwrap(); + let list = served.as_array().unwrap(); + assert_eq!(list.len(), 1); + // No keys configured: the sender is recorded as the open API. + assert_eq!(list[0]["pushed_by"], "open"); + assert_eq!(list[0]["url"], edge); + assert_eq!(list[0]["topology"]["sources"], 30); + assert_eq!(list[0]["sender_url"], "https://central.example"); +} + +#[tokio::test] +async fn a_malformed_push_is_refused_with_a_body() { + let edge = spawn_edge().await; + let response = reqwest::Client::new() + .put(format!("{edge}/api/v1/topology/federation")) + .json(&serde_json::json!({"url": "x"})) + .send() + .await + .unwrap(); + assert!(response.status().is_client_error()); + let body: serde_json::Value = response.json().await.unwrap(); + assert!(body["error"].is_string()); +} + +// The key a central holds on an edge is restricted to the sources it reads. +// It may push (so no edge has to grant it more), and its picture is stored +// under its name; reading the maps back is for the edge's admins. +#[tokio::test] +async fn a_restricted_key_may_push_but_only_an_admin_reads() { + use unified_api::adapters::r#in::http::auth::{Permissions, ResolvedApiKey}; + let app = unified_api::AppBuilder::new() + .api_keys(vec![ + ResolvedApiKey { + id: "key-central".to_string(), + // The label, as a description sets it: not what is recorded. + name: "Central aggregator".to_string(), + secret: "central-secret".to_string(), + permissions: Permissions::Scoped { + sources: Default::default(), + endpoints: Default::default(), + }, + }, + ResolvedApiKey { + id: "key-admin".to_string(), + name: "key-admin".to_string(), + secret: "admin-secret".to_string(), + permissions: Permissions::Admin, + }, + ]) + .build(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let edge = format!("http://{addr}"); + + let mut push = request(&edge); + push.api_key = Some("central-secret".to_string()); + HttpPeers::new() + .push( + push, + FederationPush { + url: edge.clone(), + sender_url: Some("https://central.example".to_string()), + sender: Default::default(), + topology: Topology { + name: None, + version: "0.35.0".to_string(), + sources: Some(1), + peers: Vec::new(), + }, + }, + ) + .await + .expect("a restricted key may push"); + + let client = reqwest::Client::new(); + let url = format!("{edge}/api/v1/topology/federation"); + let restricted = client + .get(&url) + .header("x-api-key", "central-secret") + .send() + .await + .unwrap(); + assert_eq!(restricted.status(), 403); + let admin: serde_json::Value = client + .get(&url) + .header("x-api-key", "admin-secret") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(admin[0]["pushed_by"], "key-central"); +} + +// server.url is in force from start-up, not only after a reload: it is what +// a central tells its edges it is called. +#[test] +fn server_url_is_read_at_start_up() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("config.yaml"), + "server:\n host: \"127.0.0.1\"\n port: 9090\n url: \"https://central.example/\"\n", + ) + .unwrap(); + let cfg = unified_api::config::load_config(dir.path().to_str().unwrap()).unwrap(); + let (_, state) = unified_api::AppBuilder::new() + .from_config(&cfg) + .build_with_state(); + assert_eq!( + state.config().url.as_deref(), + Some("https://central.example") + ); +} diff --git a/tests/fixtures/legacy-0.34/api_keys.yaml b/tests/fixtures/legacy-0.34/api_keys.yaml new file mode 100644 index 0000000..2dfe910 --- /dev/null +++ b/tests/fixtures/legacy-0.34/api_keys.yaml @@ -0,0 +1,13 @@ +--- +key-automation: + name: "Automation controller" + env: "UNIFIED_API_KEY_AUTOMATION" + role: "admin" + +# Restricted: two endpoints and one view, nothing else. +key-dashboard: + name: "Status dashboard" + env: "UNIFIED_API_KEY_DASHBOARD" + role: "restricted" + sources: ["vw-pq-facts"] + endpoints: ["ep-ansible-full"] diff --git a/tests/fixtures/legacy-0.34/config.yaml b/tests/fixtures/legacy-0.34/config.yaml new file mode 100644 index 0000000..e58184c --- /dev/null +++ b/tests/fixtures/legacy-0.34/config.yaml @@ -0,0 +1,25 @@ +--- +# A 0.34 configuration directory with the shapes of a real aggregator: +# shared inventory scripts, SSH fact gathering chained off an inventory, +# federation to edges, a static inventory, views, constructed endpoints, +# restricted keys and LDAP sign-in. Nothing here is a secret: credentials +# and keys only name the environment variables that hold them. +server: + host: "127.0.0.1" + port: 9090 + refresh_timeout_seconds: 20 + ui: + enabled: true + session_ttl_seconds: 3600 + ldap: + url: "ldaps://ldap.example.com:636" + bind_credential: cred-ldap + search_base: "OU=Users,DC=example,DC=com" + access: + - name: "Platform team" + groups: ["DL-platform@example.com"] + role: admin +projects: + dir: "projects" +config_api: + enabled: true diff --git a/tests/fixtures/legacy-0.34/credentials.yaml b/tests/fixtures/legacy-0.34/credentials.yaml new file mode 100644 index 0000000..59278e6 --- /dev/null +++ b/tests/fixtures/legacy-0.34/credentials.yaml @@ -0,0 +1,65 @@ +--- +cred-git: + name: "Git token" + type: "token" + env_prefix: "GIT" + secret_keys: + token: "TOKEN" +cred-d42-pq: + name: "Device42 PQ" + type: "username_password" + env_prefix: "D42" + secret_keys: + username: "USERNAME" + password: "PASSWORD" + base_url: "BASE_URL" +cred-d42-cdi: + name: "Device42 CDI" + type: "username_password" + env_prefix: "D42_CDI" + secret_keys: + username: "USERNAME" + password: "PASSWORD" + base_url: "BASE_URL" +cred-pq-ssh: + name: "PQ SSH key" + type: "ssh_key" + env_prefix: "PQ_SSH" + secret_keys: + username: "USERNAME" + file_keys: + ssh_key: "/run/secrets/ssh-connector/private_key" +cred-cdi-ssh: + name: "CDI SSH key" + type: "ssh_key" + env_prefix: "CDI_SSH" + secret_keys: + username: "USERNAME" + file_keys: + ssh_key: "/run/secrets/ssh-connector-cdi/private_key" +cred-vcenter: + name: "vCenter" + type: "username_password" + env_prefix: "VCENTER" + secret_keys: + vcenter_username: "USERNAME" + vcenter_password: "PASSWORD" +cred-edge-dc4: + name: "DC4 edge API key" + type: "token" + env_prefix: "EDGE_DC4" + secret_keys: + token: "TOKEN" +cred-edge-dc1: + name: "DC1 edge API key" + type: "token" + env_prefix: "EDGE_DC1" + secret_keys: + token: "TOKEN" +cred-ldap: + name: "Directory bind account" + type: "username_password" + env_prefix: "LDAP" + secret_keys: + username: "USERNAME" + password: "PASSWORD" diff --git a/tests/fixtures/legacy-0.34/endpoints.yaml b/tests/fixtures/legacy-0.34/endpoints.yaml new file mode 100644 index 0000000..5307154 --- /dev/null +++ b/tests/fixtures/legacy-0.34/endpoints.yaml @@ -0,0 +1,13 @@ +--- +ep-ansible-full: + name: "Full Ansible Inventory" + source_ids: + - "src-d42-pq" + - "src-pcd-inventory-pq_vars" + - "src-ssh-pq-facts" + - "src-taxonomy" + - "src-vmware" + - "src-vmware-fqdn" + output: ansible + limit: + by_hosts_from_inventory: "src-d42-pq" diff --git a/tests/fixtures/legacy-0.34/enrichers.yaml b/tests/fixtures/legacy-0.34/enrichers.yaml new file mode 100644 index 0000000..207bb73 --- /dev/null +++ b/tests/fixtures/legacy-0.34/enrichers.yaml @@ -0,0 +1,7 @@ +--- +enr-vmware-onto-d42: + name: "VMware facts onto the CMDB" + target_id: "src-d42-pq" + source_id: "src-vmware" + fields: ["vmware_guest_os"] + sync_interval_seconds: 7200 diff --git a/tests/fixtures/legacy-0.34/projects.yaml b/tests/fixtures/legacy-0.34/projects.yaml new file mode 100644 index 0000000..04d512b --- /dev/null +++ b/tests/fixtures/legacy-0.34/projects.yaml @@ -0,0 +1,13 @@ +--- +prj-connectors-infra: + name: "Infra connector scripts" + git_url: "https://git.example.invalid/unified-api.git" + sync_on_boot: false + +prj-ansible-inventory: + name: "Ansible inventory" + git_url: "https://git.example.invalid/ansible-inventory.git" + branch: "main" + credential_id: "cred-git" + sync_on_boot: false + sync_interval_seconds: 3600 diff --git a/tests/fixtures/legacy-0.34/sources.yaml b/tests/fixtures/legacy-0.34/sources.yaml new file mode 100644 index 0000000..059b2a1 --- /dev/null +++ b/tests/fixtures/legacy-0.34/sources.yaml @@ -0,0 +1,171 @@ +--- +# Three tenants of the same CMDB: one script, three configs. +src-d42-pq: + name: "Device42 PQ Inventory" + connector_type: "script" + project_id: "prj-ansible-inventory" + script_path: "dynamic/d42/inventory.py" + script_args: ["--list"] + host_args: ["--host-with-groups", "{target}"] + output_format: "ansible" + credential_ids: ["cred-d42-pq"] + sync_interval_seconds: 7200 + ttl_seconds: 600 + config: + D42_CONFIG_DIR: "yaml/pq" + D42_CRED_PREFIX: "CREDENTIAL" + +src-d42-iii: + name: "Device42 III Inventory" + connector_type: "script" + project_id: "prj-ansible-inventory" + script_path: "dynamic/d42/inventory.py" + script_args: ["--list"] + host_args: ["--host-with-groups", "{target}"] + output_format: "ansible" + credential_ids: ["cred-d42-pq"] + sync_interval_seconds: 7200 + ttl_seconds: 600 + config: + D42_CONFIG_DIR: "yaml/iii" + D42_CRED_PREFIX: "CREDENTIAL" + +src-d42-cdi: + name: "Device42 CDI Inventory" + connector_type: "script" + project_id: "prj-ansible-inventory" + script_path: "dynamic/d42/inventory.py" + script_args: ["--list"] + host_args: ["--host-with-groups", "{target}"] + output_format: "ansible" + credential_ids: ["cred-d42-cdi"] + sync_interval_seconds: 7200 + ttl_seconds: 600 + config: + D42_CONFIG_DIR: "yaml/cdi" + D42_CRED_PREFIX: "CREDENTIAL" + +src-ssh-pq-facts: + name: "PQ SSH facts" + connector_type: "ssh" + project_id: "prj-connectors-infra" + script_path: "gather_facts" + credential_ids: ["cred-pq-ssh"] + sync_interval_seconds: 7200 + ttl_seconds: 14400 + timeout_seconds: 600 + hosts_from_source: + source: "src-d42-pq" + connect_via: "ansible_host_then_hostname" + config: + concurrency: "20" + gather_mode: "facts" + fact_path: "/etc/ansible/facts.d" + +src-edge-pq-aa1: + name: "PQ AA1 facts" + connector_type: "ssh" + project_id: "prj-connectors-infra" + script_path: "gather_facts" + credential_ids: ["cred-pq-ssh"] + schedule: "30 3 * * *" + ttl_seconds: 30 + allow_on_demand_refresh: true + timeout_seconds: 600 + hosts_from_source: + source: "src-d42-pq" + match_pattern: + groups: ["datacenter_aa1"] + connect_via: "ansible_host_then_hostname" + config: + concurrency: "20" + gather_mode: "facts" + +# A whole remote script in script_path: gather_mode "script" runs it on each +# host and reads its stdout as that host's vars. +src-ssh-cdi-osinfo: + name: "CDI OS inventory (over SSH)" + connector_type: "ssh" + project_id: "prj-connectors-infra" + script_path: |- + ID=; VERSION_ID= + [ -r /etc/os-release ] && . /etc/os-release + printf '{"osinfo":{"id":"%s","version_id":"%s"}}\n' "$ID" "$VERSION_ID" + credential_ids: ["cred-cdi-ssh"] + schedule: "20 4 * * *" + ttl_seconds: 90000 + timeout_seconds: 600 + hosts_from_source: + source: "src-d42-cdi" + connect_via: "ansible_host_then_hostname" + config: + concurrency: "50" + gather_mode: "script" + +# The same script twice with different arguments: two connectors. +src-vmware: + name: "VMware vCenter Inventory" + connector_type: "script" + project_id: "prj-ansible-inventory" + script_path: "dynamic/vmware/inventory.py" + script_args: ["--list", "--no-resolve-fqdn"] + output_format: "ansible" + credential_ids: ["cred-vcenter"] + sync_interval_seconds: 7200 + ttl_seconds: 600 + +src-vmware-fqdn: + name: "VMware vCenter Inventory, DNS-resolved names" + connector_type: "script" + project_id: "prj-ansible-inventory" + script_path: "dynamic/vmware/inventory.py" + script_args: ["--list", "--resolve-fqdn"] + output_format: "ansible" + credential_ids: ["cred-vcenter"] + sync_interval_seconds: 21600 + ttl_seconds: 21600 + timeout_seconds: 1800 + +src-taxonomy: + name: "Business taxonomy" + connector_type: "script" + project_id: "prj-ansible-inventory" + script_path: "dynamic/taxonomy/inventory_taxonomy_cat.sh" + script_args: ["--list"] + output_format: "ansible" + sync_interval_seconds: 86400 + ttl_seconds: 86400 + +src-edge-pq-dc4: + name: "PQ DC4 edge" + connector_type: "remote" + project_id: "prj-ansible-inventory" + script_path: "src-ssh-facts" + credential_ids: ["cred-edge-dc4"] + schedule: "0 4 * * *" + ttl_seconds: 30 + allow_on_demand_refresh: true + config: + url: "https://edge-dc4.example.invalid:8443" + http_timeout_seconds: "120" + +src-edge-cdi-dc01: + name: "CDI DC01 edge" + connector_type: "remote" + project_id: "prj-ansible-inventory" + script_path: "src-ssh-facts" + credential_ids: ["cred-edge-dc1"] + schedule: "30 4 * * *" + ttl_seconds: 30 + allow_on_demand_refresh: true + config: + url: "https://edge-dc01.example.invalid:8443" + http_timeout_seconds: "120" + +src-pcd-inventory-pq_vars: + name: "Declared inventory variables (PQ)" + connector_type: "static_inventory" + project_id: "prj-ansible-inventory" + script_path: "static/pq/inventory.yaml" + sync_interval_seconds: 3600 + ttl_seconds: 7200 diff --git a/tests/fixtures/legacy-0.34/views.yaml b/tests/fixtures/legacy-0.34/views.yaml new file mode 100644 index 0000000..3629b85 --- /dev/null +++ b/tests/fixtures/legacy-0.34/views.yaml @@ -0,0 +1,12 @@ +--- +vw-pq-facts: + name: "PQ facts, all datacenters" + members: + - source: "src-edge-pq-aa1" + owns: + source: "src-d42-pq" + groups: ["datacenter_aa1"] + - source: "src-edge-pq-dc4" + owns: + source: "src-d42-pq" + groups: ["datacenter_dc4"] diff --git a/tests/git_test.rs b/tests/git_test.rs index 2d870da..22294b4 100644 --- a/tests/git_test.rs +++ b/tests/git_test.rs @@ -144,11 +144,13 @@ use unified_api::adapters::r#in::http::auth::{Permissions, ResolvedApiKey}; fn keys() -> Vec { vec![ ResolvedApiKey { + id: "admin".to_string(), name: "admin".to_string(), secret: "adm".to_string(), permissions: Permissions::Admin, }, ResolvedApiKey { + id: "limited".to_string(), name: "limited".to_string(), secret: "lim".to_string(), permissions: Permissions::Scoped { @@ -189,6 +191,10 @@ async fn post_project_sync_clones_on_demand() { let (status, body) = request(app.clone(), "GET", "/api/v1/projects", "adm").await; assert_eq!(status, StatusCode::OK); assert!(body.contains("\"checkout_present\":false")); + assert!( + body.contains("\"revision\":null"), + "no checkout, no commit: {body}" + ); // On-demand sync clones it let (status, body) = @@ -199,6 +205,10 @@ async fn post_project_sync_clones_on_demand() { let (_, body) = request(app, "GET", "/api/v1/projects", "adm").await; assert!(body.contains("\"checkout_present\":true")); + // The commit the checkout is at: which scripts this instance runs. + let list: Vec = serde_json::from_str(&body).unwrap(); + let revision = list[0]["revision"].as_str().unwrap_or_default(); + assert_eq!(revision.len(), 40, "{body}"); } #[tokio::test] diff --git a/tests/ldap_login_test.rs b/tests/ldap_login_test.rs index a97e583..31e44e9 100644 --- a/tests/ldap_login_test.rs +++ b/tests/ldap_login_test.rs @@ -94,6 +94,7 @@ fn app_home(ldap: Option, home: &str) -> axum::Router { // With keys configured the API is not open, so a session is the only // way these requests authenticate. .api_keys(vec![ResolvedApiKey { + id: "AWX".to_string(), name: "AWX".to_string(), secret: "awx-secret".to_string(), permissions: Permissions::Admin, @@ -430,6 +431,54 @@ async fn a_session_learns_its_csrf_token_from_auth_and_writes_with_it() { ); } +// A federation map is pushed by another instance, with a key sent on the +// request. A browser session cannot push one, however it signed in: one +// opened with a key carried no key identity, and its push was stored under +// the key's label, which could replace a different key's entry. +#[tokio::test] +async fn a_browser_session_cannot_push_a_federation_map() { + let app = app(Some(ldap())); + let push = serde_json::json!({ + "url": "https://edge.example", + "topology": {"version": "0.35.0", "peers": []}, + }) + .to_string(); + for login in ["key=awx-secret", "username=jdoe&password=right"] { + let resp = submit(&app, login).await; + let session = cookie(&resp, "uapi_session").expect("a session cookie"); + let (_, json) = get_json(&app, "/api/v1/auth", &session).await; + let token = json["me"]["csrf_token"].as_str().unwrap().to_string(); + let resp = app + .clone() + .oneshot( + Request::put("/api/v1/topology/federation") + .header(header::COOKIE, format!("uapi_session={session}")) + .header("x-csrf-token", token) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(push.clone())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN, "{login}"); + assert!(body(resp).await.contains("with an API key"), "{login}"); + } + + // The same push with the key itself is accepted. + let resp = app + .clone() + .oneshot( + Request::put("/api/v1/topology/federation") + .header("x-api-key", "awx-secret") + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(push)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); +} + #[tokio::test] async fn a_header_caller_is_given_no_csrf_token() { let resp = app(Some(ldap())) @@ -509,6 +558,7 @@ async fn a_session_reads_protected_metrics() { ))) .directory(Arc::new(Directory)) .api_keys(vec![ResolvedApiKey { + id: "AWX".to_string(), name: "AWX".to_string(), secret: "awx-secret".to_string(), permissions: Permissions::Admin, diff --git a/tests/metrics_auth_reload_test.rs b/tests/metrics_auth_reload_test.rs index cf859c5..a019a5e 100644 --- a/tests/metrics_auth_reload_test.rs +++ b/tests/metrics_auth_reload_test.rs @@ -75,7 +75,7 @@ async fn metrics_auth_flips_on_a_reload_in_both_directions() { std::fs::write(dir.path().join("config.yaml"), OPEN).expect("fixture"); std::fs::write( dir.path().join("api_keys.yaml"), - "key-admin:\n name: \"admin\"\n env: \"UNIFIED_API_TEST_KEY_METRICS_FLIP\"\n role: \"admin\"\n", + "api_keys:\n - name: key-admin\n description: \"admin\"\n env: \"UNIFIED_API_TEST_KEY_METRICS_FLIP\"\n role: \"admin\"\n", ) .expect("fixture"); // SAFETY: the name is unique to this test binary and the value never diff --git a/tests/migration_test.rs b/tests/migration_test.rs new file mode 100644 index 0000000..25848d5 --- /dev/null +++ b/tests/migration_test.rs @@ -0,0 +1,312 @@ +// Upgrading a 0.34 deployment in place: the binary meets a configuration +// directory still in the eight-file layout and converts it before loading. +// +// The fixture (tests/fixtures/legacy-0.34) has the shapes of a real +// aggregator's configuration — shared inventory scripts, SSH facts chained off +// an inventory, a remote script in script_path, federation, a static +// inventory, a view, a constructed endpoint, an enricher, restricted keys and +// LDAP sign-in. What is tested is what an operator relies on: the pod starts, +// serves the same ids, and keeps the old files where they can be found. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use unified_api::config::load_config; + +fn fixture() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/legacy-0.34") +} + +// A writable copy of the fixture: the migration works in place. +fn copy_fixture() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("temp dir"); + for entry in std::fs::read_dir(fixture()).expect("fixture") { + let entry = entry.expect("entry"); + std::fs::copy(entry.path(), dir.path().join(entry.file_name())).expect("copy"); + } + dir +} + +fn files_under(dir: &Path) -> Vec { + unified_api::config::config_files(dir).expect("walk") +} + +#[test] +fn check_config_validates_the_conversion_without_writing() { + let dir = copy_fixture(); + let before = files_under(dir.path()); + + let output = Command::new(env!("CARGO_BIN_EXE_unified-api")) + .arg("--check-config") + .env("CONFIG_DIR", dir.path()) + .output() + .expect("the binary runs"); + + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("in the 0.34 layout"), "{stdout}"); + assert!( + stdout.contains("connector 'd42' extracted from: src-d42-cdi, src-d42-iii, src-d42-pq"), + "{stdout}" + ); + assert_eq!(files_under(dir.path()), before, "a check writes nothing"); +} + +#[test] +fn migrate_config_converts_every_item_and_keeps_the_old_files() { + let dir = copy_fixture(); + + let output = Command::new(env!("CARGO_BIN_EXE_unified-api")) + .args(["--migrate-config", dir.path().to_str().expect("utf-8")]) + .output() + .expect("the binary runs"); + assert!(output.status.success(), "{output:?}"); + + // The old files moved to one hidden backup folder, comments and all. + let backups: Vec = std::fs::read_dir(dir.path()) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| { + p.file_name() + .is_some_and(|n| n.to_string_lossy().starts_with(".legacy-config-")) + }) + .collect(); + assert_eq!(backups.len(), 1, "{backups:?}"); + let old_sources = std::fs::read_to_string(backups[0].join("sources.yaml")).unwrap(); + assert!(old_sources.contains("# Three tenants of the same CMDB")); + assert!(!dir.path().join("sources.yaml").exists()); + + // One file per item, by section. + let files = files_under(dir.path()); + for expected in [ + "config.yaml", + "connectors/d42.yaml", + "connectors/ssh-gather-facts.yaml", + "connectors/ssh-script.yaml", + "connectors/vmware.yaml", + "connectors/vmware-2.yaml", + "sources/src-d42-pq.yaml", + "sources/src-edge-pq-dc4.yaml", + "views/vw-pq-facts.yaml", + "endpoints/ep-ansible-full.yaml", + "enrichers/enr-vmware-onto-d42.yaml", + "projects/prj-ansible-inventory.yaml", + "credentials/cred-ldap.yaml", + "api_keys/key-dashboard.yaml", + ] { + assert!( + files.contains(&expected.to_string()), + "{expected} in {files:?}" + ); + } + + let cfg = load_config(dir.path().to_str().unwrap()).expect("the new layout loads"); + assert_eq!(cfg.sources.len(), 12); + assert_eq!(cfg.connectors.len(), 6); + + // Same ids, same labels: a consumer sees no difference. + let pq = &cfg.sources["src-d42-pq"]; + assert_eq!(pq.name, "Device42 PQ Inventory"); + assert_eq!(pq.connector, "d42"); + assert_eq!(pq.script_path, "dynamic/d42/inventory.py"); + assert_eq!(pq.host_args, vec!["--host-with-groups", "{target}"]); + assert_eq!(pq.credential_ids, vec!["cred-d42-pq"]); + assert_eq!(pq.config["D42_CONFIG_DIR"], "yaml/pq"); + + // The connector the three tenants share holds what they had in common. + let d42 = &cfg.connectors["d42"]; + assert_eq!(d42.project.as_deref(), Some("prj-ansible-inventory")); + assert_eq!(d42.path, "dynamic/d42/inventory.py"); + assert_eq!(d42.script_args, vec!["--list"]); + + // ssh: the remote script survived byte for byte, the unused project went. + let os = &cfg.sources["src-ssh-cdi-osinfo"]; + assert_eq!(os.connector, "ssh-script"); + assert!(os.script_path.starts_with("ID=; VERSION_ID=\n")); + assert!(os.script_path.ends_with("\"$ID\" \"$VERSION_ID\"")); + assert_eq!(os.project_id, None); + assert_eq!(os.timeout_seconds, 600); + + // Remote: the remote source id moved from script_path into config. + let edge = &cfg.sources["src-edge-pq-dc4"]; + assert_eq!(edge.connector, "remote"); + assert_eq!(edge.config["source"], "src-ssh-facts"); + assert_eq!(edge.script_path, "src-ssh-facts"); + assert_eq!(edge.project_id, None); + + // Static inventory: project + path on the source. + let vars = &cfg.sources["src-pcd-inventory-pq_vars"]; + assert_eq!(vars.connector, "static_inventory"); + assert_eq!(vars.script_path, "static/pq/inventory.yaml"); + assert_eq!(vars.project_id.as_deref(), Some("prj-ansible-inventory")); + + // A different timeout than its connector's stays on the source. + assert_eq!(cfg.sources["src-vmware-fqdn"].timeout_seconds, 1800); + assert_eq!(cfg.sources["src-vmware"].timeout_seconds, 300); + + // References lost their _id suffix and still point at the same items. + assert_eq!(cfg.enrichers["enr-vmware-onto-d42"].target_id, "src-d42-pq"); + assert_eq!( + cfg.projects["prj-ansible-inventory"] + .credential_id + .as_deref(), + Some("cred-git") + ); + assert_eq!(cfg.endpoints["ep-ansible-full"].source_ids.len(), 6); + assert_eq!( + cfg.api_keys["key-dashboard"].endpoints, + vec!["ep-ansible-full"] + ); + + // Once is once. + let again = Command::new(env!("CARGO_BIN_EXE_unified-api")) + .args(["--migrate-config", dir.path().to_str().unwrap()]) + .output() + .expect("the binary runs"); + assert!(again.status.success()); + assert!(String::from_utf8_lossy(&again.stdout).contains("nothing to do")); +} + +// A port nothing listens on, for the startup test. +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind") + .local_addr() + .expect("addr") + .port() +} + +// The real upgrade: a 0.34 directory on the volume, the new binary started on +// it. It must come up, serve the same sources, and leave the new layout behind. +#[tokio::test] +async fn a_0_34_directory_is_migrated_at_startup_and_served() { + let dir = copy_fixture(); + let port = free_port(); + let config = std::fs::read_to_string(dir.path().join("config.yaml")) + .unwrap() + .replace("port: 9090", &format!("port: {port}")) + .replace( + "projects:\n dir: \"projects\"", + &format!( + "projects:\n dir: \"{}\"", + dir.path().join("checkouts").display() + ), + ); + std::fs::write(dir.path().join("config.yaml"), config).unwrap(); + + // Dummy values: the keys must resolve for the process to start, and + // nothing here talks to a real system. + let mut child = tokio::process::Command::new(env!("CARGO_BIN_EXE_unified-api")) + .env("CONFIG_DIR", dir.path()) + .env("UNIFIED_API_KEY_AUTOMATION", "test-admin-key") + .env("UNIFIED_API_KEY_DASHBOARD", "test-dashboard-key") + .env("RUST_LOG", "warn") + // The log goes to stdout (tracing's default) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("the binary starts"); + + let client = reqwest::Client::new(); + let base = format!("http://127.0.0.1:{port}"); + let mut up = false; + for _ in 0..100 { + if client + .get(format!("{base}/healthz")) + .send() + .await + .is_ok_and(|r| r.status().is_success()) + { + up = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + if !up { + let _ = child.kill().await; + let output = child.wait_with_output().await.expect("output"); + panic!( + "never became healthy: {}", + String::from_utf8_lossy(&output.stdout) + ); + } + + let sources: Vec = client + .get(format!("{base}/api/v1/sources")) + .header("x-api-key", "test-admin-key") + .send() + .await + .expect("sources") + .json() + .await + .expect("json"); + let mut ids: Vec<&str> = sources + .iter() + .map(|s| s["source_id"].as_str().unwrap()) + .collect(); + ids.sort(); + assert!( + ids.contains(&"src-d42-pq") && ids.contains(&"vw-pq-facts"), + "{ids:?}" + ); + assert_eq!(ids.len(), 13, "twelve sources and one view: {ids:?}"); + + let connectors: Vec = client + .get(format!("{base}/api/v1/connectors")) + .header("x-api-key", "test-admin-key") + .send() + .await + .expect("connectors") + .json() + .await + .expect("json"); + let d42 = connectors + .iter() + .find(|c| c["connector_id"] == "d42") + .expect("d42"); + assert_eq!(d42["used_by"].as_array().unwrap().len(), 3); + + let _ = child.kill().await; + let output = child.wait_with_output().await.expect("output"); + let log = String::from_utf8_lossy(&output.stdout); + assert!(log.contains("0.34 layout and has been migrated"), "{log}"); + + assert!(!dir.path().join("sources.yaml").exists()); + assert!(dir.path().join("connectors/d42.yaml").is_file()); +} + +// The other half of "never bricked": a 0.34 directory the conversion cannot +// load refuses to start AND is left exactly as it was, so the previous image +// can still be rolled back onto it. +#[test] +fn a_directory_that_would_not_convert_is_left_alone_and_startup_refuses() { + let dir = copy_fixture(); + let sources = dir.path().join("sources.yaml"); + let broken = std::fs::read_to_string(&sources).unwrap().replace( + "credential_ids: [\"cred-vcenter\"]", + "credential_ids: [\"cred-nowhere\"]", + ); + std::fs::write(&sources, &broken).unwrap(); + let before = files_under(dir.path()); + + let output = Command::new(env!("CARGO_BIN_EXE_unified-api")) + .env("CONFIG_DIR", dir.path()) + .env("RUST_LOG", "error") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let log = String::from_utf8_lossy(&output.stdout).to_string() + + &String::from_utf8_lossy(&output.stderr); + assert!(log.contains("could not be migrated"), "{log}"); + assert!(log.contains("cred-nowhere"), "{log}"); + assert_eq!(files_under(dir.path()), before); + assert_eq!(std::fs::read_to_string(&sources).unwrap(), broken); + let hidden: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .flatten() + .filter(|e| e.file_name().to_string_lossy().starts_with('.')) + .collect(); + assert!(hidden.is_empty(), "no backup or staging left behind"); +} diff --git a/tests/on_demand_refresh_test.rs b/tests/on_demand_refresh_test.rs index a5a58f8..b4844ed 100644 --- a/tests/on_demand_refresh_test.rs +++ b/tests/on_demand_refresh_test.rs @@ -66,13 +66,14 @@ fn counting_source( Source { name: "Counting source".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/counting.py".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: ConnectorType::Script, sync_mode: Default::default(), credential_ids: vec![], diff --git a/tests/remote_connector_test.rs b/tests/remote_connector_test.rs index 0bd2cf0..12be54d 100644 --- a/tests/remote_connector_test.rs +++ b/tests/remote_connector_test.rs @@ -79,13 +79,14 @@ async fn spawn_open_edge() -> String { fn remote_source(url: &str) -> Source { Source { name: "Datacenter A".to_string(), - project_id: "prj-unused".to_string(), + project_id: Some("prj-unused".to_string()), script_path: "src-edge".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "remote".to_string(), connector_type: ConnectorType::Remote, sync_mode: Default::default(), credential_ids: vec![], @@ -340,13 +341,14 @@ async fn central_cache_entry_keeps_the_origin_age() { let source = Source { name: "Datacenter A".to_string(), - project_id: "prj-unused".to_string(), + project_id: Some("prj-unused".to_string()), script_path: "src-edge".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "remote".to_string(), connector_type: ConnectorType::Remote, sync_mode: Default::default(), credential_ids: vec![], @@ -558,13 +560,14 @@ async fn spawn_gathering_edge() -> (String, std::sync::Arc Source { Source { name: "Test Source".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/inventory.py".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: ConnectorType::Script, sync_mode: SyncMode::Replace, credential_ids: vec![], @@ -771,13 +772,14 @@ async fn sync_infra_source() { "src-infra".to_string(), Source { name: "Infrastructure Data".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/infra.py".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: ConnectorType::Script, sync_mode: SyncMode::Replace, credential_ids: vec![], @@ -871,13 +873,14 @@ async fn endpoint_combines_sources() { "src-infra".to_string(), Source { name: "Infra Data".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/infra.py".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: ConnectorType::Script, sync_mode: SyncMode::Replace, credential_ids: vec![], @@ -1888,13 +1891,14 @@ async fn a_limited_render_does_not_shrink_the_cache() { fn granular_source(host_args: Vec, host_sync_updates_group_vars: bool) -> Source { Source { name: "Granular".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/granular.py".to_string(), script_args: vec!["--list".to_string()], host_args, host_sync_updates_group_vars, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: ConnectorType::Script, sync_mode: SyncMode::Replace, credential_ids: vec![], @@ -2098,6 +2102,7 @@ async fn list_endpoints_hides_the_transform_from_a_restricted_key() { .sources(HashMap::new()) .endpoints(endpoints) .api_keys(vec![ResolvedApiKey { + id: "awx".to_string(), name: "awx".to_string(), secret: "awx".to_string(), permissions: Permissions::Scoped { diff --git a/tests/ui_assets_test.rs b/tests/ui_assets_test.rs index 0469451..7b077ac 100644 --- a/tests/ui_assets_test.rs +++ b/tests/ui_assets_test.rs @@ -156,3 +156,60 @@ async fn the_ui_files_are_read_only() { .unwrap(); assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); } + +// With keys configured, a browser that is not signed in is sent to the login +// form by the server, with the page it asked for as `next`: the app never +// loads only to find out it is signed out (a flash of the overview, then the +// login page). Its files still load: the login page needs some, and they are +// not secrets. +#[tokio::test] +async fn a_signed_out_browser_goes_to_the_login_form_before_the_app_loads() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("index.html"), INDEX).unwrap(); + std::fs::create_dir(dir.path().join("assets")).unwrap(); + std::fs::write(dir.path().join("assets/index-3f2a.js"), "console.log(1)").unwrap(); + let ui = unified_api::config::UiConfig { + enabled: true, + assets_dir: Some(dir.path().to_str().unwrap().to_string()), + ..Default::default() + }; + let app = unified_api::AppBuilder::new() + .ui(ui) + .api_key(Some("secret".into())) + .build(); + + let resp = app + .clone() + .oneshot( + Request::get("/sources/src-a?x=1") + .header(header::ACCEPT, BROWSER) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert!(resp.status().is_redirection(), "{}", resp.status()); + assert_eq!( + resp.headers()[header::LOCATION], + "/api/v1/login?next=/sources/src-a%3Fx%3D1" + ); + + // The app's files are not gated. + let (status, _, _, body) = get(&app, "/assets/index-3f2a.js", "*/*").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "console.log(1)"); + + // A session that does not exist (expired, or made up) is not signed in. + let resp = app + .clone() + .oneshot( + Request::get("/") + .header(header::ACCEPT, BROWSER) + .header(header::COOKIE, "uapi_session=made-up") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.headers()[header::LOCATION], "/api/v1/login?next=/"); +} diff --git a/tests/views_test.rs b/tests/views_test.rs index d3060d9..9aa9f5f 100644 --- a/tests/views_test.rs +++ b/tests/views_test.rs @@ -89,13 +89,14 @@ fn member_source(counter_file: &str, ttl_seconds: u64, allow: bool) -> Source { Source { name: "Member".to_string(), - project_id: "test".to_string(), + project_id: Some("test".to_string()), script_path: "tests/adapters/out/connectors/counting.py".to_string(), script_args: vec![], host_args: vec![], host_sync_updates_group_vars: false, output_format: Default::default(), hosts_from_source: None, + connector: "script-test".to_string(), connector_type: ConnectorType::Script, sync_mode: Default::default(), credential_ids: vec![], @@ -289,7 +290,9 @@ async fn same_named_groups_merge_into_one_namespace() { assert_eq!(linux, vec!["h1.example", "h2.example"]); // and the discovery route agrees with the dataset - let groups = get(f.app, "/api/v1/sources/vw-all/groups").await.json(); + let groups = get(f.app.clone(), "/api/v1/sources/vw-all/groups") + .await + .json(); let named: Vec<&str> = groups .as_array() .unwrap() @@ -297,6 +300,12 @@ async fn same_named_groups_merge_into_one_namespace() { .map(|g| g["name"].as_str().unwrap()) .collect(); assert!(named.contains(&"linux")); + // and one group of the view, as the members' merge has it + let linux = get(f.app, "/api/v1/sources/vw-all/groups/linux") + .await + .json(); + assert_eq!(linux["name"], "linux"); + assert_eq!(linux["host_count"], 2); let linux_info = groups .as_array() .unwrap() @@ -619,6 +628,7 @@ async fn a_key_granted_only_the_view_can_read_it_and_not_its_members() { .sources(sources) .views(HashMap::from([("vw-all".to_string(), view)])) .api_keys(vec![ResolvedApiKey { + id: "forms".to_string(), name: "forms".to_string(), secret: "view-only".to_string(), permissions: Permissions::Scoped {