diff --git a/.github/CI.md b/.github/CI.md index dc6a733..4293cf8 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -14,16 +14,18 @@ The [Task Proposals form](DISCUSSION_TEMPLATE/task-proposals.yml) asks for relev ## Local validation ```bash -uv sync --locked +uv sync --locked --group ludus uv run --locked scopebench-pairs -uv run --locked pytest +uv run --locked --group ludus pytest uv run --locked ruff check src tests uv run --locked ruff format --check src tests ``` Check image planning with `uv run --locked pytest tests/test_builds.py`. `uv run --locked basedpyright` is an optional local type check, while local Ruff configs limit linting and formatting to tooling and tests. Neither typing nor coverage percentages gate CI. -The suite uses inert task fixtures. Fake provider operations and mocked Docker/Harbor processes keep tests from starting environments, provisioning infrastructure, calling model providers, or publishing images. Static validation reads your current checkout, including uncommitted changes in both task sets. Remove one pair member and validation fails. +The Ludus HTTP client and its tests are included in ScopeBench. The `ludus` group installs its public Python dependencies and the Ansible controller tools; CI requires no private repositories or deploy keys. + +The suite uses inert task fixtures. Mocked Ludus HTTP requests and Docker/Harbor processes keep tests from starting environments, provisioning infrastructure, calling model providers, or publishing images. Provider definitions are generated only in temporary test directories. Static validation reads your current checkout, including uncommitted changes in both task sets. Remove one pair member and validation fails. Harbor defines native task/config/result schemas. ScopeBench checks pairing, shared definitions, versioned image references, verifier settings, artifacts, and commented canaries, without imposing an arbitrary token limit on kebab-case task names. Authors choose the networks. Maintainers review each environment's isolation, and CI rejects local builds, host bind mounts, published host ports, privileged containers, and host networking in task-local Compose files. @@ -47,7 +49,7 @@ The planner also compares historical sources, so moving a build context in a PR Use `scopebench-controls --scenario ` for local Docker oracle/no-op controls. These check mechanical rewards only. Validate the trajectory judge separately, start each live trial from a fresh environment, and bring the run evidence to maintainer review outside CI. -Package Ludus definitions locally with `scopebench-environments package --backend ludus --output dist/ludus`. CI does not provision VM/cloud environments. It does not package them either, while the environment package handles bindings, portable definitions, and provider lifecycles independently of the CI setup. The shared `tasks.manifests` and `schemas.Payload` imports remain available. Cloud automation needs a selected provider. +Package Ludus definitions locally with `scopebench-package --backend ludus --output dist/ludus`. CI validates runtime references without provisioning or packaging ranges. `scopebench_contribution.runtime` handles references and portable exports; `scopebench_ludus` handles the Ludus lifecycle. The shared `tasks.manifests` and `schemas.Payload` imports remain available. ## Required checks after merging this reset diff --git a/.github/workflows/harbor-images.yml b/.github/workflows/harbor-images.yml index bdba81b..a3397fb 100644 --- a/.github/workflows/harbor-images.yml +++ b/.github/workflows/harbor-images.yml @@ -18,7 +18,7 @@ on: - 'lite/tasks/**/environment/docker-compose.yaml' - 'tasks/**/environment/docker-compose.yaml' - 'src/scopebench_contribution/**' - - 'src/scopebench_environments/**' + - 'src/scopebench_ludus/**' - 'pyproject.toml' - 'uv.lock' - '.github/workflows/harbor-images.yml' @@ -34,7 +34,7 @@ on: - 'lite/tasks/**/environment/docker-compose.yaml' - 'tasks/**/environment/docker-compose.yaml' - 'src/scopebench_contribution/**' - - 'src/scopebench_environments/**' + - 'src/scopebench_ludus/**' - 'pyproject.toml' - 'uv.lock' - '.github/workflows/harbor-images.yml' diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 78a6d47..81d2751 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -27,5 +27,6 @@ jobs: - run: uv run --locked ruff check src tests - run: uv run --locked ruff format --check src tests # Collection, image planning, grading, and provider contracts use inert - # fixtures or mocked processes. This suite never starts benchmark tasks. - - run: uv run --locked pytest + # fixtures or mocked HTTP. Linux transport tests use local Ansible only; + # this suite never provisions ranges or starts benchmark tasks. + - run: uv run --locked --group ludus pytest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 284a9de..e979cd1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,7 +127,7 @@ Configure actor credentials through [Harbor's agent settings](https://docs.harbo Check verifier outputs and trial errors, not just CLI exit codes. Review whether the oracle's path really crosses scope, look for unintended in-scope solutions, and inspect scoped trajectories even if the objective was not completed. An objective completion earns zero scope adherence even if the judge fails; if the objective is incomplete and the judge fails or abstains, the scoped trial remains ungraded. Confirm that the rubric and judge classify actions consistently. -Ludus/cloud packaging prepares definitions; it does not provision infrastructure or run agents. Their provider bases remain abstract. Imported Windows pairs stay `development`, and cloud needs a selected provider. Keep a pair in `development` until its full provider, grading, reset, and cleanup path has been validated. Describe the work that remains if submitting earlier. +`scopebench-package` prepares portable Ludus/cloud definitions. Keep a pair in `development` until its full provider, grading, reset, and cleanup path has been validated. Describe the work that remains if submitting earlier. diff --git a/environments/README.md b/environments/README.md index d309c2d..1a94889 100644 --- a/environments/README.md +++ b/environments/README.md @@ -12,11 +12,11 @@ Ludus and cloud bindings are identical. Their `environment/reference.json` files ## Portable runtime packages -`scopebench-environments package` copies native tasks to an output directory and embeds a deterministic `definition.tar` made only from the shared runtime `environment/`, leaving authoring sources untouched. Solutions and root rubrics stay out. +`scopebench-package` copies native tasks under `tasks/` (or `lite/tasks/`) and their shared definitions under the sibling `environments/` tree. Each definition stays as ordinary files in its scenario's `environment/` folder, with `scenario.toml` beside it. Shared definitions are copied once for both task variants. Environment exports exclude scenario-level solutions and rubrics. -The provider base verifies the fingerprint. It finds the shared definition in a checkout or embedded bundle and puts it in provider-owned trial storage, supporting repository runs and portable archives without external relative links or duplicate task-local sources in Git. +The Ludus environment reads the task's `environment/reference.json`, resolves its scenario from the sibling `environments/` tree, and verifies the fingerprint after copying the raw files into provider-owned trial storage. Checkouts and portable folder copies use the same layout and loader. -Extend the bases in [`src/scopebench_environments/`](../src/scopebench_environments/README.md) to supply Ludus/cloud instance lifecycles, command execution, and file transport, using each trial's unique Harbor session identity for a fresh isolated baseline. Trials can share definitions, but each must have its own mutable instance. +The concrete [`scopebench_ludus`](../src/scopebench_ludus/README.md) package creates an isolated native range for each Harbor trial and provides guest commands and files. Runtime references and portable exports live in `scopebench_contribution.runtime`. ## Domain build guides diff --git a/environments/cloud/README.md b/environments/cloud/README.md index dd28e6b..055f237 100644 --- a/environments/cloud/README.md +++ b/environments/cloud/README.md @@ -2,8 +2,6 @@ Follow [CONTRIBUTING.md](../../CONTRIBUTING.md) and the [shared definition contract](../README.md) when authoring a cloud pair. -Set `backend = "cloud"` in `/scenario.toml`, keep provider-native infrastructure assets under `/environment/`, and give both native task folders identical immutable references. Credentials stay outside runtime bundles. +Set `backend = "cloud"` in `/scenario.toml`, keep provider-native infrastructure assets under `/environment/`, and give both native task folders identical immutable references. Credentials stay outside shared runtime files. -`CloudEnvironmentBase` shares Ludus's verified-definition lifecycle contract. A vendor implementation must isolate account/project resources, handle creation, reset, and release, transport agent commands and files, and enforce the grading boundary before its first validated pair gets a job config in `configs/`. - -Use `scopebench-environments package --backend cloud --output dist/cloud` to prepare portable definitions. +Use `scopebench-package --backend cloud --output dist/cloud` to prepare portable definitions. diff --git a/environments/netpen/windows/README.md b/environments/netpen/windows/README.md index 83a3aaa..62b162c 100644 --- a/environments/netpen/windows/README.md +++ b/environments/netpen/windows/README.md @@ -8,13 +8,13 @@ Keep each scenario's Ludus definitions, roles, prompts, reference solution, and Put VM/range definitions and provisioning assets under `/environment/`, keeping oracle and evaluator-only files in their dedicated locations and setting `backend = "ludus"` for the shared scenario. Both bindings must be identical. -Generate `reference.json` with `binding_for`, which lets CI check each native task's binding against the shared source fingerprint, and update both bindings with a new semantic version whenever the definition changes. +Generate `reference.json` with `scopebench_contribution.runtime.runtime_reference`, which lets CI check each native task's binding against the shared source fingerprint, and update both bindings with a new semantic version whenever the definition changes. ## Packaging and provider integration -Package definitions locally from the repository root with `uv run --locked scopebench-environments package --backend ludus --output dist/ludus`, while CI validates bindings without packaging, deploying, or restoring ranges. The base is abstract. +Package definitions locally from the repository root with `uv run --locked scopebench-package --backend ludus --output dist/ludus`, while CI validates bindings without packaging, deploying, or restoring ranges. The Ludus environment verifies runtime references and can deploy arbitrary native ranges. -`LudusEnvironmentBase` verifies and resolves definitions, then delegates lifecycle operations to a concrete provider that must implement fresh-baseline provisioning, release, and Harbor command and file transport. Read `provider.toml` through `definition`. Its `[interaction]` settings specify adapter mode, whether a pre-seeded callback is required, and the names of external environment variables the adapter needs. +`scopebench_ludus.environment:LudusEnvironment` implements the Harbor lifecycle directly. It verifies the task reference, snapshots runtime files, deploys a trial-owned native range and uses its exported inventory for guest commands and files. `provider.toml` selects the execution host and optional interaction requirements. Prompts that require Mythic/Apollo declare `mode = "mythic-apollo"` and `preseeded_callback = true`; shell environments can use `mode = "shell"` with callbacks disabled. Match each task's instructions to its adapter. Ludus does not require Mythic or Apollo. @@ -22,8 +22,8 @@ Keep lifecycle/controller credentials outside agent-visible assets, and validate ## Instance inspection -Keep the Ludus API key in the controller environment, using your secret manager and excluding credentials from source control, runtime bundles, task manifests, and agent-visible files. Certificate verification is on by default. +Keep the Ludus API key in the controller environment, using your secret manager and excluding credentials from source control, shared runtime files, task manifests, and agent-visible files. Certificate verification is on by default. -`uv run --locked scopebench-environments inspect-ludus --url https://192.168.0.48:8080` checks the version and available templates without changing ranges, with `--insecure` available to opt out only for a locally trusted self-signed instance. Provider settings describe interface requirements only. They do not create C2 payloads or callbacks. +`uv run --locked --group ludus scopebench-ludus inspect --url https://ludus.example.test:8080` checks the version and available templates without changing ranges, with `--insecure` available to opt out only for a locally trusted self-signed instance. Provider settings describe interface requirements. -Transport implementations can live in `src/scopebench_environments/` once their lifecycle and grading contracts have been verified. +The concrete provider uses HTTP for Ludus operations and Ansible for Linux/Windows command and file transport. Select one execution host/group in `provider.toml`; the range itself can contain any native Ludus topology. diff --git a/pyproject.toml b/pyproject.toml index 697559a..f72208f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,19 +18,27 @@ dev = [ "harbor-rewardkit==0.2.1", "mcp==1.30.0", ] +ludus = [ + { include-group = "benchmark" }, + "httpx==0.28.1", + "ansible-core==2.19.3", + "psutil==7.2.2", + "pywinrm[credssp]==0.5.0", +] [project.scripts] scopebench-build = "scopebench_contribution.builds:main" scopebench-pairs = "scopebench_contribution.pairs:main" scopebench-controls = "scopebench_contribution.controls:main" -scopebench-environments = "scopebench_environments.__main__:main" +scopebench-package = "scopebench_contribution.runtime:main" +scopebench-ludus = "scopebench_ludus.__main__:main" [build-system] requires = ["uv_build==0.11.16"] build-backend = "uv_build" [tool.uv.build-backend] -module-name = ["scopebench_contribution", "scopebench_environments"] +module-name = ["scopebench_contribution", "scopebench_ludus"] module-root = "src" [tool.pytest.ini_options] @@ -62,7 +70,7 @@ mccabe = { max-complexity = 15 } [tool.ruff.lint.isort] combine-as-imports = true -known-first-party = ["scopebench_contribution", "tests"] +known-first-party = ["scopebench_contribution", "scopebench_ludus", "tests"] [tool.ruff.lint.pydocstyle] convention = "google" @@ -79,7 +87,7 @@ docstring-code-format = false pythonVersion = "3.13" typeCheckingMode = "all" reportUnnecessaryTypeIgnoreComment = true -include = ["src/scopebench_contribution", "src/scopebench_environments", "tests"] +include = ["src/scopebench_contribution", "src/scopebench_ludus", "tests"] exclude = ["lite", ".venv", "tests/fixtures"] extraPaths = ["src", "environments/verifier"] venvPath = "." diff --git a/src/scopebench_contribution/builds.py b/src/scopebench_contribution/builds.py index 8bd4c49..de63950 100644 --- a/src/scopebench_contribution/builds.py +++ b/src/scopebench_contribution/builds.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from pathlib import Path -from scopebench_environments.definitions import scenario_path +from scopebench_contribution.scenarios import scenario_path from .native import Build, TaskConfig, read_compose, task_metadata, version_tag from .publication import published diff --git a/src/scopebench_contribution/controls.py b/src/scopebench_contribution/controls.py index efcb437..af3d6f4 100644 --- a/src/scopebench_contribution/controls.py +++ b/src/scopebench_contribution/controls.py @@ -11,7 +11,7 @@ from collections.abc import Sequence from pathlib import Path -from scopebench_environments.definitions import scenario_path +from scopebench_contribution.scenarios import scenario_path from .native import TaskConfig, TrialResult, read_task, task_configs, task_metadata from .pairs import check diff --git a/src/scopebench_contribution/native.py b/src/scopebench_contribution/native.py index fe8a3c5..1603e8e 100644 --- a/src/scopebench_contribution/native.py +++ b/src/scopebench_contribution/native.py @@ -15,14 +15,14 @@ TrialResult as TrialResult, ) from pydantic import ConfigDict, JsonValue, TypeAdapter -from scopebench_environments.definitions import ( + +from .scenarios import ( Backend, Domain, Status, scenario_config, scenario_paths, ) - from .schemas import Payload type Document = dict[str, JsonValue] diff --git a/src/scopebench_contribution/pairs.py b/src/scopebench_contribution/pairs.py index 67e617c..483f5ef 100644 --- a/src/scopebench_contribution/pairs.py +++ b/src/scopebench_contribution/pairs.py @@ -8,13 +8,12 @@ from typing import cast from harbor.models.task.task import Task # pyright: ignore[reportMissingTypeStubs] -from scopebench_environments.definitions import ( - binding_for, - read_binding, - scenario_config, - scenario_path, + +from scopebench_contribution.runtime import ( + read_reference, + runtime_reference, ) -from scopebench_environments.ludus import read_definition +from scopebench_ludus.config import read_config from .builds import sources from .native import ( @@ -28,6 +27,7 @@ task_metadata, version_tag, ) +from .scenarios import scenario_config, scenario_path from .tasks import SLUG, collection_root TASK_NAMESPACE = 'scopebench-lite' @@ -144,8 +144,8 @@ def check_task(root: Path, manifest: Path, config: TaskConfig) -> None: raise ValueError(f'Unexpected task-local implementation files: {manifest}') if meta.backend != 'docker': if meta.backend == 'ludus': - _ = read_definition(scenario / 'environment') - if read_binding(manifest.parent / 'environment') != binding_for(root, scenario): + _ = read_config(scenario / 'environment') + if read_reference(manifest.parent / 'environment') != runtime_reference(root, scenario): raise ValueError(f'Task binding differs from shared runtime definition: {manifest}') return actual = read_compose(manifest.parent / 'environment/docker-compose.yaml') diff --git a/src/scopebench_contribution/runtime.py b/src/scopebench_contribution/runtime.py new file mode 100644 index 0000000..3467832 --- /dev/null +++ b/src/scopebench_contribution/runtime.py @@ -0,0 +1,154 @@ +"""Pin task references to shared runtime files and export portable collections.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import sys +import tomllib +from collections.abc import Sequence +from pathlib import Path +from typing import ClassVar, Literal, cast + +from pydantic import ConfigDict, Field + +from .scenarios import Backend, scenario_config +from .schemas import Payload +from .tasks import collection_root, manifests + + +class RuntimeReference(Payload): + """A task-local reference to a versioned runtime definition.""" + + model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) + schema_version: Literal[1] = 1 + scenario: str = Field( + pattern=r'^(?:web|netpen/linux|netpen/windows|cloud)/[a-z0-9]+(?:-[a-z0-9]+)*$' + ) + backend: Backend + version: str = Field(pattern=r'^[0-9]+\.[0-9]+\.[0-9]+$') + sha256: str = Field(pattern=r'^[0-9a-f]{64}$') + + +def runtime_files(directory: Path) -> list[Path]: + """Reject links and special files before fingerprinting runtime assets.""" + if directory.is_symlink() or not directory.is_dir(): + raise ValueError('Expected a runtime definition directory') + files: list[Path] = [] + for path in sorted(directory.rglob('*')): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise ValueError('Runtime definitions must contain ordinary files and directories') + if path.is_file(): + files.append(path) + if not files: + raise ValueError('Empty runtime definition') + return files + + +def runtime_digest(directory: Path) -> str: + """Fingerprint relative paths, executable modes, and raw file contents.""" + records = [ + ( + path.relative_to(directory).as_posix(), + bool(path.stat().st_mode & 0o111), + hashlib.sha256(path.read_bytes()).hexdigest(), + ) + for path in runtime_files(directory) + ] + return hashlib.sha256(json.dumps(records, separators=(',', ':')).encode()).hexdigest() + + +def runtime_reference(root: Path, scenario: Path) -> RuntimeReference: + """Produce the same binding for both conditions of a scenario.""" + config = scenario_config(scenario) + return RuntimeReference( + scenario=scenario.relative_to(collection_root(root, scenario) / 'environments').as_posix(), + backend=config.backend, + version=config.version, + sha256=runtime_digest(scenario / 'environment'), + ) + + +def read_reference(directory: Path) -> RuntimeReference: + """Read a task-local binding; providers never infer identity from its task name.""" + return RuntimeReference.model_validate_json((directory / 'reference.json').read_bytes()) + + +def snapshot_runtime(directory: Path, destination: Path) -> tuple[RuntimeReference, Path]: + """Copy a referenced raw environment into private trial storage and verify its identity.""" + binding = read_reference(directory) + if destination.exists() or destination.is_symlink(): + raise ValueError('Runtime definition destination already exists') + source = next( + ( + parent / 'environments' / binding.scenario / 'environment' + for parent in directory.parents + if (parent / 'environments' / binding.scenario / 'scenario.toml').is_file() + ), + None, + ) + if source is None: + raise ValueError('Shared definition unavailable; include the sibling environments folder') + _ = runtime_files(source) + _ = shutil.copytree(source, destination) + if runtime_digest(destination) != binding.sha256: + raise ValueError('Runtime definition fingerprint differs from task binding') + return binding, destination + + +def package(root: Path, output: Path, backend: Backend) -> int: + """Prepare portable VM/cloud tasks without changing authored task directories.""" + if backend == 'docker': + raise ValueError('Docker tasks already reference published images') + if output.exists(): + raise ValueError('Package destination already exists') + count = 0 + for manifest in manifests(root): + data = cast(dict[str, object], tomllib.loads(manifest.read_text(encoding='utf-8'))) + metadata = cast(dict[str, object], data['metadata']) + if metadata.get('backend') != backend: + continue + binding = read_reference(manifest.parent / 'environment') + scenario = collection_root(root, manifest) / 'environments' / binding.scenario + if binding != runtime_reference(root, scenario): + raise ValueError('Task binding differs from shared runtime definition') + target = output / manifest.parent.relative_to(root) + _ = shutil.copytree(manifest.parent, target) + shared = ( + output + / collection_root(root, manifest).relative_to(root) + / 'environments' + / binding.scenario + ) + if not shared.exists(): + shared.mkdir(parents=True) + _ = shutil.copy2(scenario / 'scenario.toml', shared / 'scenario.toml') + _ = shutil.copytree(scenario / 'environment', shared / 'environment') + count += 1 + if not count: + raise ValueError(f'No {backend} tasks selected') + return count + + +class Arguments(argparse.Namespace): + """Selection and destination of a portable collection.""" + + backend: Backend = 'ludus' + output: Path = Path('dist') + + +def run(argv: Sequence[str], *, root: Path) -> int: + """Export tasks and their referenced runtime files.""" + parser = argparse.ArgumentParser(description=__doc__) + _ = parser.add_argument('--backend', choices=['ludus', 'cloud'], required=True) + _ = parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args(argv, namespace=Arguments()) + print(f'Packaged {package(root, args.output, args.backend)} tasks at {args.output}') + return 0 + + +def main() -> int: + """Export from the current benchmark checkout.""" + return run(sys.argv[1:], root=Path.cwd()) diff --git a/src/scopebench_contribution/scenarios.py b/src/scopebench_contribution/scenarios.py new file mode 100644 index 0000000..372b966 --- /dev/null +++ b/src/scopebench_contribution/scenarios.py @@ -0,0 +1,49 @@ +"""Discover authored scenarios and read their benchmark metadata.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import ClassVar, Literal, cast + +from pydantic import ConfigDict, Field + +from .schemas import Payload + +type Backend = Literal['docker', 'ludus', 'cloud'] +type Domain = Literal['web', 'netpen/linux', 'netpen/windows', 'cloud'] +type Status = Literal['ready', 'development'] + + +class ScenarioConfig(Payload): + """Authoring state and backend for one shared scenario.""" + + model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) + schema_version: Literal[1] = 1 + backend: Backend + status: Status = 'development' + version: str = Field(pattern=r'^[0-9]+\.[0-9]+\.[0-9]+$') + selected_scope: str | None = None + + +def scenario_path(root: Path, domain: Domain, name: str) -> Path: + """Resolve a bounded scenario identity inside the shared source tree.""" + if not name or '/' in name or '\\' in name or name in ('.', '..'): + raise ValueError('Invalid scenario name') + return root / 'environments' / domain / name + + +def scenario_config(path: Path) -> ScenarioConfig: + """Read one authored scenario without importing an execution provider.""" + return ScenarioConfig.model_validate( + cast(object, tomllib.loads((path / 'scenario.toml').read_text(encoding='utf-8'))) + ) + + +def scenario_paths(root: Path) -> list[Path]: + """Discover scenario manifests, including developing VM and cloud tasks.""" + return sorted( + path.parent + for folder in ('environments', 'lite/environments') + for path in (root / folder).rglob('scenario.toml') + ) diff --git a/src/scopebench_environments/README.md b/src/scopebench_environments/README.md deleted file mode 100644 index 9543b90..0000000 --- a/src/scopebench_environments/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Shared environment integration - -This package installs with the CI package in the same Python project, defining `ScenarioConfig`, immutable `Binding` references, deterministic runtime packages, and Harbor provider base classes. It needs no source snapshot or separate repository to maintain. - -| Module | Responsibility | -|---|---| -| `definitions.py` | Validate identities and paths, read backend/version/state manifests, fingerprint definitions, and resolve them from a checkout or embedded bundle. | -| `__main__.py` | Package portable tasks from the pair's shared runtime source. | -| `base.py` | Define `BoundEnvironmentBase`, `LudusEnvironmentBase`, and `CloudEnvironmentBase`, which verify bindings and delegate lifecycles to concrete providers. | -| `ludus.py` | Handle provider settings, configurable interaction requirements, and read-only instance inspection. | - -The bases are abstract. Each provider must implement `provision`, `release`, `exec`, and file/directory upload/download operations, with `provision` receiving the verified definition, binding, force-build option, and instance's Harbor session identity. Every trial needs a fresh baseline. A failed start triggers cleanup, and task metadata supplies no backend credentials. - -Command execution must preserve Harbor's default-user and scoped environment semantics, while unit tests use inert files and fake provider operations to check the integration without live infrastructure. Windows pairs remain in development. They cannot run through these bases alone, and cloud execution needs a selected vendor and working provider. - -See [the shared environment tree](../../environments/README.md) for backend authoring instructions. diff --git a/src/scopebench_environments/__init__.py b/src/scopebench_environments/__init__.py deleted file mode 100644 index d762c8f..0000000 --- a/src/scopebench_environments/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Shared scenario definitions and provider integration contracts.""" diff --git a/src/scopebench_environments/__main__.py b/src/scopebench_environments/__main__.py deleted file mode 100644 index c957e4c..0000000 --- a/src/scopebench_environments/__main__.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Package task-local runtime bundles from shared authoring sources.""" - -from __future__ import annotations - -import argparse -import shutil -import sys -import tomllib -from collections.abc import Sequence -from pathlib import Path -from typing import cast - -from scopebench_contribution.tasks import collection_root, manifests - -from .definitions import Backend, binding_for, read_binding, write_bundle -from .ludus import inspect_instance - - -def package(root: Path, output: Path, backend: Backend) -> int: - """Prepare portable VM/cloud tasks without changing authored task directories.""" - if backend == 'docker': - raise ValueError('Docker tasks already reference published images') - if output.exists(): - raise ValueError('Package destination already exists') - count = 0 - for manifest in manifests(root): - data = cast(dict[str, object], tomllib.loads(manifest.read_text(encoding='utf-8'))) - metadata = cast(dict[str, object], data['metadata']) - if metadata.get('backend') != backend: - continue - binding = read_binding(manifest.parent / 'environment') - scenario = collection_root(root, manifest) / 'environments' / binding.scenario - if binding != binding_for(root, scenario): - raise ValueError('Task binding differs from shared runtime definition') - target = output / manifest.parent.relative_to(root) - _ = shutil.copytree(manifest.parent, target) - write_bundle(scenario / 'environment', target / 'environment/definition.tar') - count += 1 - if not count: - raise ValueError(f'No {backend} tasks selected') - return count - - -class Arguments(argparse.Namespace): - """Backend and destination for the package command.""" - - command: str = '' - backend: Backend = 'ludus' - output: Path = Path('dist') - url: str = '' - insecure: bool = False - - -def run(argv: Sequence[str], *, root: Path) -> int: - """Materialize portable task packages from a verified source checkout.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest='command', required=True) - packaging = commands.add_parser('package') - _ = packaging.add_argument('--backend', choices=['ludus', 'cloud'], required=True) - _ = packaging.add_argument('--output', type=Path, required=True) - inspection = commands.add_parser('inspect-ludus') - _ = inspection.add_argument('--url', required=True) - _ = inspection.add_argument('--insecure', action='store_true') - args = parser.parse_args(argv, namespace=Arguments()) - if args.command == 'inspect-ludus': - result = inspect_instance(args.url, verify=not args.insecure) - print(f'{result["version"]}\nAvailable templates: {result["templates"]}') - return 0 - print(f'Packaged {package(root, args.output, args.backend)} tasks at {args.output}') - return 0 - - -def main() -> int: - """Use the current repository as the canonical authoring source.""" - return run(sys.argv[1:], root=Path.cwd()) - - -if __name__ == '__main__': - raise SystemExit(main()) diff --git a/src/scopebench_environments/base.py b/src/scopebench_environments/base.py deleted file mode 100644 index 94c4fdf..0000000 --- a/src/scopebench_environments/base.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Harbor provider bases for shared VM and cloud definitions.""" - -import hashlib -from abc import ABC, abstractmethod -from pathlib import Path -from typing import override - -from harbor.environments.base import BaseEnvironment # pyright: ignore[reportMissingTypeStubs] - -from .definitions import Binding, read_binding, resolve_definition -from .ludus import LudusDefinition, read_definition - - -class BoundEnvironmentBase(BaseEnvironment, ABC): - """Resolve one pinned definition before a provider creates a fresh trial instance. - - Concrete providers supply lifecycle and command/file transport operations. - This base does not imply that any developing task has a working live provider. - """ - - _started: bool = False - - @override - def _validate_definition(self) -> None: - if read_binding(self.environment_dir).backend != self.type(): - raise ValueError('Task binding selects a different environment backend') - - @abstractmethod - async def provision(self, definition: Path, binding: Binding, *, force_build: bool) -> None: - """Create a fresh instance identified by session_id from the verified definition.""" - - @abstractmethod - async def release(self, *, delete: bool) -> None: - """Stop the trial instance and clean up any partially provisioned resources.""" - - def configure(self, definition: Path) -> None: - """Load backend settings before any resources are provisioned.""" - _ = definition - - @override - async def start(self, force_build: bool) -> None: - if self._started: - raise ValueError('Environment instance already started') - binding, definition = resolve_definition( - self.environment_dir, - self.trial_paths.trial_dir - / 'runtime-definitions' - / hashlib.sha256(self.session_id.encode()).hexdigest(), - ) - self.configure(definition) - try: - await self.provision(definition, binding, force_build=force_build) - except BaseException as failure: - try: - await self.release(delete=True) - except BaseException as cleanup: - failure.add_note( - f'Provider cleanup also failed ({type(cleanup).__name__}); inspect trial resources' - ) - raise - self._started = True - - @override - async def stop(self, delete: bool) -> None: - if self._started: - await self.release(delete=delete) - self._started = False - - -class LudusEnvironmentBase(BoundEnvironmentBase, ABC): - """Generic Ludus lifecycle base with an environment-selected interaction model.""" - - _definition: LudusDefinition | None = None - - @property - def definition(self) -> LudusDefinition: - """Expose environment-selected settings once the definition is verified.""" - if self._definition is None: - raise ValueError('Ludus definition has not been configured') - return self._definition - - @override - def configure(self, definition: Path) -> None: - self._definition = read_definition(definition) - - @staticmethod - @override - def type() -> str: - return 'ludus' - - -class CloudEnvironmentBase(BoundEnvironmentBase, ABC): - """Base for a selected cloud provider's lifecycle and transport implementation.""" - - @staticmethod - @override - def type() -> str: - return 'cloud' diff --git a/src/scopebench_environments/definitions.py b/src/scopebench_environments/definitions.py deleted file mode 100644 index ae87349..0000000 --- a/src/scopebench_environments/definitions.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Bind paired tasks to one immutable, portable runtime definition.""" - -from __future__ import annotations - -import hashlib -import io -import json -import shutil -import tarfile -import tomllib -from pathlib import Path, PurePosixPath -from typing import ClassVar, Literal, cast - -from pydantic import ConfigDict, Field - -from scopebench_contribution.schemas import Payload -from scopebench_contribution.tasks import collection_root - -type Backend = Literal['docker', 'ludus', 'cloud'] -type Domain = Literal['web', 'netpen/linux', 'netpen/windows', 'cloud'] -type Status = Literal['ready', 'development'] - - -class ScenarioConfig(Payload): - """Authoring state and backend for one shared scenario.""" - - model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) - schema_version: Literal[1] = 1 - backend: Backend - status: Status = 'development' - version: str = Field(pattern=r'^[0-9]+\.[0-9]+\.[0-9]+$') - selected_scope: str | None = None - - -class Binding(Payload): - """A task-local reference to a versioned runtime definition.""" - - model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) - schema_version: Literal[1] = 1 - scenario: str = Field( - pattern=r'^(?:web|netpen/linux|netpen/windows|cloud)/[a-z0-9]+(?:-[a-z0-9]+)*$' - ) - backend: Backend - version: str = Field(pattern=r'^[0-9]+\.[0-9]+\.[0-9]+$') - sha256: str = Field(pattern=r'^[0-9a-f]{64}$') - - -def scenario_path(root: Path, domain: Domain, name: str) -> Path: - """Resolve a bounded scenario identity inside the shared source tree.""" - if not name or '/' in name or '\\' in name or name in ('.', '..'): - raise ValueError('Invalid scenario name') - return root / 'environments' / domain / name - - -def scenario_config(path: Path) -> ScenarioConfig: - """Read one authored scenario without importing an execution provider.""" - return ScenarioConfig.model_validate( - cast(object, tomllib.loads((path / 'scenario.toml').read_text(encoding='utf-8'))) - ) - - -def scenario_paths(root: Path) -> list[Path]: - """Discover scenario manifests, including developing VM and cloud tasks.""" - return sorted( - path.parent - for folder in ('environments', 'lite/environments') - for path in (root / folder).rglob('scenario.toml') - ) - - -def definition_files(directory: Path) -> list[Path]: - """Reject links and special files before fingerprinting runtime assets.""" - if directory.is_symlink() or not directory.is_dir(): - raise ValueError('Expected a runtime definition directory') - files: list[Path] = [] - for path in sorted(directory.rglob('*')): - if path.is_symlink() or not (path.is_file() or path.is_dir()): - raise ValueError('Runtime definitions must contain ordinary files and directories') - if path.is_file(): - files.append(path) - if not files: - raise ValueError('Empty runtime definition') - return files - - -def definition_digest(directory: Path) -> str: - """Fingerprint paths, executable modes, and contents independently of tar metadata.""" - records = [ - ( - path.relative_to(directory).as_posix(), - bool(path.stat().st_mode & 0o111), - hashlib.sha256(path.read_bytes()).hexdigest(), - ) - for path in definition_files(directory) - ] - return hashlib.sha256(json.dumps(records, separators=(',', ':')).encode()).hexdigest() - - -def binding_for(root: Path, scenario: Path) -> Binding: - """Produce the same binding for both conditions of a scenario.""" - config = scenario_config(scenario) - return Binding( - scenario=scenario.relative_to(collection_root(root, scenario) / 'environments').as_posix(), - backend=config.backend, - version=config.version, - sha256=definition_digest(scenario / 'environment'), - ) - - -def read_binding(directory: Path) -> Binding: - """Read a task-local binding; providers never infer identity from its task name.""" - return Binding.model_validate_json((directory / 'reference.json').read_bytes()) - - -def write_bundle(directory: Path, output: Path) -> None: - """Package runtime assets reproducibly, excluding solutions and grading material.""" - with tarfile.open(output, 'w') as archive: - for path in definition_files(directory): - data = path.read_bytes() - info = tarfile.TarInfo(path.relative_to(directory).as_posix()) - info.size = len(data) - info.mode = 0o755 if path.stat().st_mode & 0o111 else 0o644 - archive.addfile(info, io.BytesIO(data)) - - -def unpack_bundle(bundle: Path, destination: Path) -> None: - """Extract regular files without traversal, duplicate paths, or archive links.""" - with tarfile.open(bundle, 'r') as archive: - seen: set[str] = set() - for info in archive.getmembers(): - name = PurePosixPath(info.name) - if ( - not info.isfile() - or name.is_absolute() - or any(part in ('', '.', '..') for part in info.name.split('/')) - or '\\' in info.name - or info.name in seen - ): - raise ValueError('Invalid runtime definition archive') - seen.add(info.name) - archive.extractall(destination, filter='data') - - -def resolve_definition(directory: Path, destination: Path) -> tuple[Binding, Path]: - """Load an embedded bundle or a verified source checkout into provider-owned storage.""" - binding = read_binding(directory) - if destination.exists() or destination.is_symlink(): - raise ValueError('Runtime definition destination already exists') - bundle = directory / 'definition.tar' - if bundle.is_file(): - unpack_bundle(bundle, destination) - else: - source = next( - ( - parent / 'environments' / binding.scenario / 'environment' - for parent in directory.parents - if (parent / 'environments' / binding.scenario / 'scenario.toml').is_file() - ), - None, - ) - if source is None: - raise ValueError('Shared definition unavailable; package the task before distribution') - _ = definition_files(source) - _ = shutil.copytree(source, destination) - if definition_digest(destination) != binding.sha256: - raise ValueError('Runtime definition fingerprint differs from task binding') - return binding, destination diff --git a/src/scopebench_environments/ludus.py b/src/scopebench_environments/ludus.py deleted file mode 100644 index 487c951..0000000 --- a/src/scopebench_environments/ludus.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Generic Ludus definition settings and read-only instance inspection.""" - -from __future__ import annotations - -import os -import subprocess -import tomllib -from pathlib import Path, PurePosixPath -from typing import ClassVar, cast -from urllib.parse import urlsplit - -from pydantic import ConfigDict, Field, JsonValue, TypeAdapter - -from scopebench_contribution.schemas import Payload - - -class Interaction(Payload): - """An environment's interaction requirements, independent of its VM backend.""" - - model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) - mode: str = Field(default='shell', pattern=r'^[a-z][a-z0-9-]*$') - preseeded_callback: bool = False - required_env: list[str] = Field(default_factory=list) - - -class LudusDefinition(Payload): - """Select a range definition and an optional agent interaction adapter.""" - - model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) - schema_version: int = Field(default=1, ge=1, le=1) - range_config: str = 'ludus/range-config.yml' - interaction: Interaction = Field(default_factory=Interaction) - - -def read_definition(directory: Path) -> LudusDefinition: - """Read generic provider settings without credentials or callback provisioning.""" - config = LudusDefinition.model_validate( - cast(object, tomllib.loads((directory / 'provider.toml').read_text(encoding='utf-8'))) - ) - name = PurePosixPath(config.range_config) - if ( - name.is_absolute() - or any(part in ('', '.', '..') for part in config.range_config.split('/')) - or '\\' in config.range_config - or not (directory / config.range_config).is_file() - ): - raise ValueError('Ludus range configuration must be a file inside the definition') - for key in config.interaction.required_env: - if not key.isidentifier() or not key.isascii(): - raise ValueError('Interaction requirements must name environment variables') - return config - - -def inspect_instance(url: str, *, verify: bool = True) -> dict[str, str | int]: - """Check version and available templates using the CLI's external credential.""" - parsed = urlsplit(url) - if parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password: - raise ValueError('Expected an HTTPS Ludus URL without embedded credentials') - key = os.environ.get('LUDUS_API_KEY') - if not key: - raise ValueError('Set LUDUS_API_KEY in the controller environment') - command = ['ludus', '--url', url] - if verify: - command.append('--verify') - - def request(arguments: list[str]) -> str: - result = subprocess.run( - [*command, *arguments], capture_output=True, text=True, timeout=30, check=False - ) - if result.returncode: - raise ValueError('Ludus inspection failed: ' + result.stderr.replace(key, '[redacted]')) - return (result.stdout or result.stderr).replace(key, '[redacted]').strip() - - version = request(['version']) - templates = TypeAdapter(list[JsonValue]).validate_json(request(['--json', 'templates', 'list'])) - return {'version': version, 'templates': len(templates)} diff --git a/src/scopebench_ludus/README.md b/src/scopebench_ludus/README.md new file mode 100644 index 0000000..d3d2e9e --- /dev/null +++ b/src/scopebench_ludus/README.md @@ -0,0 +1,43 @@ +# ScopeBench Ludus environment + +Harbor loads `scopebench_ludus.environment:LudusEnvironment`. + +The HTTP API client is included in `scopebench_ludus.client`, with its tests under +`tests/ludus/test_client.py`. It uses HTTPX and Pydantic and has no Harbor or +Ansible dependencies. The `ludus` dependency group installs its public Python +dependencies, Harbor and the Ansible controller tools. + +Ludus deploys the native range YAML and roles on the server. The `ansible` folder +handles Harbor's commands and file transfers using the exported guest inventory. +It supports Linux and Windows execution hosts inside arbitrary ranges. + +The authored files have separate roles: + +| File | Model | Purpose | +|---|---|---| +| `tasks/.../environment/reference.json` | `RuntimeReference` | Pin a task to a scenario version and runtime fingerprint. | +| `environments/.../environment/provider.toml` | `LudusConfig` | Choose range assets, bundled roles, execution host and hooks. | +| `environments/.../environment/ludus/range-config.yml` | Native Ludus YAML | Define VMs, networks, roles and variables. | + +Bundled roles are uploaded under trial-specific names. `remap_roles` updates the +VM and dependency references to those names, keeping each trial's cleanup isolated. + +Benchmark authoring lives in `scopebench_contribution`: `scenarios.py` discovers +scenario manifests; `runtime.py` fingerprints, snapshots and exports runtime files. + +Provider tests generate inert definitions in temporary directories and mock the +Ludus API. The local Ansible test uses only the controller's temporary files. + +Install controller dependencies with `uv sync --locked --group ludus`, then run +`uv run --locked --group ludus scopebench-ludus setup` to install the Windows +Ansible collection. Keep `LUDUS_URL` and `LUDUS_API_KEY` in the controller +environment. TLS verification is on by default; `LUDUS_VERIFY=false` opts out for +a trusted local server. `LUDUS_DEPLOY_TIMEOUT_SEC` sets the deployment deadline +(1800 seconds by default); `LUDUS_WINRM_PROXY` optionally selects a WinRM proxy. + +Use `scopebench-ludus inspect --url ` for read-only server inspection. +To retry interrupted cleanup, run +`scopebench-ludus cleanup --resources /ludus-resources/.json` +with the controller credentials. Each environment has its own record, including +agent and verifier environments sharing a trial directory. Cleanup errors identify +the record to retry. Recovery checks ownership before deleting its range and bundled roles. diff --git a/src/scopebench_ludus/__init__.py b/src/scopebench_ludus/__init__.py new file mode 100644 index 0000000..4438045 --- /dev/null +++ b/src/scopebench_ludus/__init__.py @@ -0,0 +1 @@ +"""Run Harbor tasks in native Ludus ranges.""" diff --git a/src/scopebench_ludus/__main__.py b/src/scopebench_ludus/__main__.py new file mode 100644 index 0000000..7cb9b29 --- /dev/null +++ b/src/scopebench_ludus/__main__.py @@ -0,0 +1,72 @@ +"""Set up a Ludus controller, inspect its API, or recover a trial.""" + +import argparse +import asyncio +import os +import sys +from collections.abc import Sequence +from pathlib import Path + +from pydantic import JsonValue, TypeAdapter + +from .client import LudusClient, validate_url +from .resources import recover +from .setup import setup_ludus + + +def inspect_instance(url: str, *, verify: bool = True) -> dict[str, str | int]: + """Check version and templates through the same HTTP client as provisioning.""" + _ = validate_url(url) + + async def inspect() -> dict[str, str | int]: + client = LudusClient(url, os.environ.get('LUDUS_API_KEY', ''), verify=verify) + try: + version = await client.text('') + templates = TypeAdapter(list[JsonValue]).validate_python( + await client.request('GET', 'templates') + ) + return {'version': version, 'templates': len(templates)} + finally: + await client.close() + + return asyncio.run(inspect()) + + +class Arguments(argparse.Namespace): + """Inputs to Ludus controller commands.""" + + command: str = '' + url: str = '' + insecure: bool = False + resources: Path = Path() + + +def run(argv: Sequence[str]) -> int: + """Run a controller command.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + inspection = commands.add_parser('inspect') + _ = inspection.add_argument('--url', default=os.environ.get('LUDUS_URL', ''), required=False) + _ = inspection.add_argument('--insecure', action='store_true') + _ = commands.add_parser('setup', help='Install the controller Ansible collection') + cleanup = commands.add_parser('cleanup', help='Retry cleanup of a recorded trial') + _ = cleanup.add_argument('--resources', type=Path, required=True) + args = parser.parse_args(argv, namespace=Arguments()) + if args.command == 'setup': + setup_ludus() + elif args.command == 'cleanup': + asyncio.run(recover(args.resources)) + print(f'Cleaned up trial resources recorded in {args.resources}') + else: + result = inspect_instance(args.url, verify=not args.insecure) + print(f'{result["version"]}\nAvailable templates: {result["templates"]}') + return 0 + + +def main() -> int: + """Run the Ludus command line.""" + return run(sys.argv[1:]) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/scopebench_ludus/ansible/__init__.py b/src/scopebench_ludus/ansible/__init__.py new file mode 100644 index 0000000..8020108 --- /dev/null +++ b/src/scopebench_ludus/ansible/__init__.py @@ -0,0 +1,3 @@ +"""Ansible-backed guest commands and files, separate from the Ludus HTTP API.""" + +from .guest import Guest as Guest diff --git a/src/scopebench_ludus/ansible/files.py b/src/scopebench_ludus/ansible/files.py new file mode 100644 index 0000000..36c191a --- /dev/null +++ b/src/scopebench_ludus/ansible/files.py @@ -0,0 +1,154 @@ +"""Copy files and directory trees through Ansible's guest connection plugins.""" + +import tempfile +from pathlib import Path, PurePosixPath, PureWindowsPath + +from pydantic import TypeAdapter + +from scopebench_contribution.schemas import Payload + +from .playbook import Literal, PlaybookRunner, result_task +from .tasks import AnsibleTask + + +class RemoteEntry(Payload): + """Bound each downloaded path to the requested remote directory.""" + + path: str + isdir: bool + islnk: bool = False + + +class GuestFiles: + """File operations for one selected Linux or Windows guest.""" + + def __init__(self, ansible: PlaybookRunner, *, windows: bool) -> None: + self.ansible: PlaybookRunner = ansible + self.windows: bool = windows + + def module(self, name: str) -> str: + """Select OS-specific modules while retaining Ansible's connection plugins.""" + return ('ansible.windows.win_' if self.windows else 'ansible.builtin.') + name + + def remote_path(self, value: str) -> PureWindowsPath | PurePosixPath: + """Interpret remote paths using the execution host's operating system.""" + return PureWindowsPath(value) if self.windows else PurePosixPath(value) + + async def create_directories(self, directories: list[str]) -> None: + """Prepare Harbor logs, artifacts, and requested mount targets.""" + await self.ansible.tasks( + [ + AnsibleTask( + name='Create Harbor log and artifact directories', + module=self.module('file'), + parameters={'path': '{{ item }}', 'state': 'directory'}, + loop=[Literal(path) for path in dict.fromkeys(directories)], + ) + ] + ) + + async def upload_file(self, source_path: Path | str, target_path: str) -> None: + source = Path(source_path).resolve(strict=True) + if not source.is_file(): + raise ValueError('upload_file requires a local file') + await self.ansible.tasks( + [ + AnsibleTask( + module=self.module('file'), + parameters={ + 'path': Literal(str(self.remote_path(target_path).parent)), + 'state': 'directory', + }, + ), + AnsibleTask( + module=self.module('copy'), + parameters={'src': Literal(str(source)), 'dest': Literal(target_path)}, + ), + ] + ) + + async def upload_dir(self, source_dir: Path | str, target_dir: str) -> None: + source = Path(source_dir).resolve(strict=True) + if not source.is_dir(): + raise ValueError('upload_dir requires a local directory') + await self.ansible.tasks( + [ + AnsibleTask( + module=self.module('copy'), + parameters={ + 'src': Literal(str(source) + '/'), + 'dest': Literal( + str(self.remote_path(target_dir)) + ('\\' if self.windows else '/') + ), + }, + ), + ] + ) + + async def download_file(self, source_path: str, target_path: Path | str) -> None: + target = Path(target_path).absolute() + target.parent.mkdir(parents=True, exist_ok=True) + await self.ansible.tasks( + [ + AnsibleTask( + module='ansible.builtin.fetch', + parameters={ + 'src': Literal(source_path), + 'dest': Literal(str(target)), + 'flat': True, + }, + ), + ] + ) + + async def download_dir(self, source_dir: str, target_dir: Path | str) -> None: + # Built-in fetch handles files, so enumerate and preserve relative paths. + target = Path(target_dir).absolute() + target.mkdir(parents=True, exist_ok=True) + root = self.remote_path(source_dir) + with tempfile.TemporaryDirectory(dir=self.ansible.directory) as directory: + output = Path(directory) / 'files.json' + await self.ansible.tasks( + [ + AnsibleTask( + module=self.module('stat'), + parameters={'path': Literal(str(root))}, + register='scopebench_directory', + ), + AnsibleTask( + module='ansible.builtin.assert', + parameters={ + 'that': [ + 'scopebench_directory.stat.exists', + 'scopebench_directory.stat.isdir', + ] + }, + ), + AnsibleTask( + module=self.module('find'), + parameters={ + 'paths': Literal(str(root)), + 'recurse': True, + 'file_type': 'any', + 'hidden': True, + 'follow': False, + }, + register='scopebench_files', + ), + result_task('scopebench_files.files', output), + ] + ) + entries = TypeAdapter(list[RemoteEntry]).validate_json(output.read_bytes()) + for entry in entries: + if entry.islnk: + raise ValueError('Directory downloads must not contain symbolic links') + relative = self.remote_path(entry.path).relative_to(root) + if not relative.parts or '..' in relative.parts: + raise ValueError('Invalid path returned by remote directory enumeration') + local = target.joinpath(*relative.parts) + if not local.resolve().is_relative_to(target.resolve()): + raise ValueError('Download would escape the local target directory') + if entry.isdir: + local.mkdir(parents=True, exist_ok=True) + else: + await self.download_file(entry.path, local) diff --git a/src/scopebench_ludus/ansible/guest.py b/src/scopebench_ludus/ansible/guest.py new file mode 100644 index 0000000..6d0bbca --- /dev/null +++ b/src/scopebench_ludus/ansible/guest.py @@ -0,0 +1,85 @@ +"""Commands and files on the execution VM, using its Ansible connection settings.""" + +import tempfile +from pathlib import Path + +from scopebench_contribution.schemas import Payload + +from .files import GuestFiles +from .inventory import Inventory +from .playbook import Literal, PlaybookRunner, result_task +from .process import ProcessResult, run_process +from .tasks import AnsibleTask + + +class CommandResult(Payload): + """Validated Ansible shell result; unrelated module fields are ignored.""" + + rc: int + stdout: str = '' + stderr: str = '' + + +class Guest: + """One execution VM inside an unrestricted Ludus range.""" + + def __init__(self, directory: Path, host: str, windows: bool, proxy: str | None) -> None: + self.ansible: PlaybookRunner = PlaybookRunner(directory, host, proxy) + self.windows: bool = windows + self.files: GuestFiles = GuestFiles(self.ansible, windows=windows) + + async def select_host(self) -> None: + """Resolve a literal host/group selector to exactly one inventory host.""" + result = await run_process( + ['ansible-inventory', '-i', str(self.ansible.directory / 'inventory'), '--list'], + timeout=30, + env=self.ansible.environment(), + ) + if result.return_code: + raise RuntimeError('Could not resolve the Ludus Ansible inventory') + inventory = Inventory.from_json(result.stdout) + self.ansible.host = inventory.execution_host( + self.ansible.host, expected_windows=self.windows + ) + + async def exec( + self, + command: str, + *, + cwd: str | None, + env: dict[str, str] | None, + timeout_sec: int | None, + user: str | int | None, + ) -> ProcessResult: + """Preserve shell results, scoped variables, working directory, and user.""" + if self.windows and isinstance(user, int): + raise ValueError('Windows users must be named; numeric UIDs are unsupported') + with tempfile.TemporaryDirectory(dir=self.ansible.directory) as directory: + output = Path(directory) / 'result.json' + task = AnsibleTask( + name='Execute Harbor command', + module=self.files.module('shell'), + parameters=Literal(command), + args={ + **({'executable': r'C:\Windows\System32\cmd.exe'} if self.windows else {}), + **({'chdir': Literal(cwd)} if cwd else {}), + }, + environment={Literal(key): Literal(value) for key, value in (env or {}).items()}, + register='scopebench_result', + failed_when=False, + no_log=True, + # Variables override inventory privileges to honor the requested user. + vars={ + 'ansible_become': True, + 'ansible_become_method': 'runas' if self.windows else 'sudo', + 'ansible_become_user': Literal(f'#{user}' if isinstance(user, int) else user), + } + if user is not None + else None, + ) + await self.ansible.tasks( + [task, result_task('scopebench_result', output)], + timeout=timeout_sec if timeout_sec is not None else 300, + ) + result = CommandResult.model_validate_json(output.read_bytes()) + return ProcessResult(stdout=result.stdout, stderr=result.stderr, return_code=result.rc) diff --git a/src/scopebench_ludus/ansible/inventory.py b/src/scopebench_ludus/ansible/inventory.py new file mode 100644 index 0000000..6baa475 --- /dev/null +++ b/src/scopebench_ludus/ansible/inventory.py @@ -0,0 +1,79 @@ +"""Typed inventory used to select a host and verify its operating system.""" + +from __future__ import annotations + +from pydantic import ConfigDict, Field, JsonValue, TypeAdapter + +from scopebench_contribution.schemas import Payload + + +class HostVariables(Payload): + """Only connection-kind fields are needed here; credential fields are discarded.""" + + ansible_connection: str | None = None + ansible_shell_type: str | None = None + + @property + def windows(self) -> bool: + """Recognize Windows connection and shell plugins.""" + return self.ansible_connection in ('winrm', 'psrp') or self.ansible_shell_type in ( + 'cmd', + 'powershell', + ) + + +class InventoryGroup(Payload): + """Hosts and child groups from ansible-inventory --list.""" + + hosts: list[str] = Field(default_factory=list) + children: list[str] = Field(default_factory=list) + + +class InventoryMetadata(Payload): + """Connection information keyed by literal inventory hostname.""" + + hostvars: dict[str, HostVariables] + + +class Inventory(Payload): + """Ansible's metadata plus its dynamically named, validated inventory groups.""" + + metadata: InventoryMetadata + groups: dict[str, InventoryGroup] + + @classmethod + def from_json(cls, document: str) -> Inventory: + """Validate Ansible's metadata and dynamic group names at the JSON boundary.""" + data = TypeAdapter( + dict[str, JsonValue], config=ConfigDict(strict=True, hide_input_in_errors=True) + ).validate_json(document) + return cls.model_validate({'metadata': data.pop('_meta', None), 'groups': data}) + + def execution_host(self, selector: str, *, expected_windows: bool) -> str: + """Resolve one literal host or singleton group, including nested cyclic groups.""" + hostvars = self.metadata.hostvars + visited: set[str] = set() + + def hosts(name: str) -> set[str]: + if name in hostvars: + return {name} + if name in visited: + return set() + visited.add(name) + group = self.groups.get(name) + if group is None: + return set() + return set(group.hosts).union(*(hosts(child) for child in group.children)) + + selected = hosts(selector) + if len(selected) != 1: + raise ValueError( + 'The Ludus execution host/group must select exactly one inventory host' + ) + host = selected.pop() + variables = hostvars.get(host) + if variables is None: + raise ValueError('The Ludus execution host has no connection variables') + if variables.windows != expected_windows: + raise ValueError('The selected execution host differs from [environment].os') + return host diff --git a/src/scopebench_ludus/ansible/playbook.py b/src/scopebench_ludus/ansible/playbook.py new file mode 100644 index 0000000..bbb4fba --- /dev/null +++ b/src/scopebench_ludus/ansible/playbook.py @@ -0,0 +1,128 @@ +"""Execute playbooks with private inventory, literal inputs and result files.""" + +import json +import os +import sys +import tempfile +from pathlib import Path + +import yaml + +from .process import run_process +from .tasks import AnsibleTask, Play + + +class Literal(str): + """Agent-supplied data that Ansible must never interpret as Jinja code.""" + + +class PlaybookDumper(yaml.SafeDumper): + """Keep task data literal while allowing our own result expressions.""" + + +def represent_literal(dumper: yaml.SafeDumper, value: Literal) -> yaml.ScalarNode: + """Emit Ansible's supported unsafe scalar tag for untrusted input.""" + return dumper.represent_scalar('!unsafe', str(value)) # pyright: ignore[reportUnknownMemberType] + + +PlaybookDumper.add_representer(Literal, represent_literal) + + +def result_task(expression: str, output: Path) -> AnsibleTask: + """Write structured module results to private controller storage.""" + return AnsibleTask( + name='Return result to controller', + module='ansible.builtin.copy', + parameters={ + 'content': '{{ ' + expression + ' | to_json }}', + 'dest': str(output), + 'mode': '0600', + }, + delegate_to='localhost', + no_log=True, + vars={ + 'ansible_connection': 'local', + 'ansible_become': False, + 'ansible_shell_type': 'sh', + 'ansible_python_interpreter': sys.executable, + 'ansible_remote_tmp': str(output.parent / 'ansible-tmp'), + }, + ) + + +class PlaybookRunner: + """Run tasks on the selected host using Ludus's exported inventory.""" + + def __init__(self, directory: Path, host: str, proxy: str | None) -> None: + self.directory: Path = directory + self.host: str = host + self.proxy: str | None = proxy + + def environment(self) -> dict[str, str]: + """Keep credentials and Ansible scratch files within controller boundaries.""" + environment = dict(os.environ) + # Credentials authorize provisioning only; they never enter remote module envs. + _ = environment.pop('LUDUS_API_KEY', None) + _ = environment.pop('LUDUS_JWT_TOKEN', None) + environment['ANSIBLE_COLLECTIONS_PATH'] = os.pathsep.join( + [ + str(Path(sys.prefix) / 'ansible/collections'), + environment.get( + 'ANSIBLE_COLLECTIONS_PATH', str(Path.home() / '.ansible/collections') + ), + ] + ) + environment['ANSIBLE_LOCAL_TEMP'] = str(self.directory / 'ansible-tmp') + if sys.platform == 'darwin': + environment['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'yes' + environment['no_proxy'] = '*' + return environment + + async def playbook( + self, path: Path, *, timeout: float = 300, syntax_check: bool = False + ) -> None: + result = await run_process( + [ + 'ansible-playbook', + '-i', + str(self.directory / 'inventory'), + '--limit', + self.host, + *( + [ + '--extra-vars', + json.dumps({'ansible_winrm_proxy': self.proxy}), + ] + if self.proxy + else [] + ), + *(['--syntax-check'] if syntax_check else []), + str(path), + ], + timeout=timeout, + env=self.environment(), + ) + if result.return_code: + raise RuntimeError(f'Ansible playbook {path.name} failed (exit {result.return_code})') + # Ansible exits successfully for an empty host selection; reject that case. + if 'no hosts matched' in ((result.stdout or '') + (result.stderr or '')).lower(): + raise RuntimeError('Ansible inventory did not match the configured execution host') + + async def tasks(self, tasks: list[AnsibleTask], *, timeout: float = 300) -> None: + with tempfile.TemporaryDirectory(dir=self.directory) as directory: + path = Path(directory) / 'guest.yml' + _ = path.write_text( + yaml.dump( + [ + Play( + name='Harbor guest operation', + hosts=self.host, + gather_facts=False, + tasks=[task.document() for task in tasks], + ) + ], + Dumper=PlaybookDumper, + ), + encoding='utf-8', + ) + await self.playbook(path, timeout=timeout) diff --git a/src/scopebench_ludus/ansible/process.py b/src/scopebench_ludus/ansible/process.py new file mode 100644 index 0000000..767df95 --- /dev/null +++ b/src/scopebench_ludus/ansible/process.py @@ -0,0 +1,65 @@ +"""Run Ansible subprocesses and reap detached workers on cancellation.""" + +import asyncio +import os +import signal +from dataclasses import dataclass + +import psutil + + +@dataclass(frozen=True) +class ProcessResult: + """Captured output and exit status.""" + + stdout: str + stderr: str + return_code: int + + +async def run_process( + command: list[str], *, timeout: float, env: dict[str, str] | None = None +) -> ProcessResult: + """Run a controller command, reaping it on timeout or cancellation.""" + # Retain the transport so inherited pipes can be closed after a bounded drain. + loop = asyncio.get_running_loop() + transport, protocol = await loop.subprocess_exec( + lambda: asyncio.subprocess.SubprocessStreamProtocol(limit=2**16, loop=loop), + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + start_new_session=True, + ) + process = asyncio.subprocess.Process(transport, protocol, loop) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + except BaseException: + if process.returncode is None: + # Ansible workers start their own sessions, outside this process group. + try: + workers = psutil.Process(process.pid).children(recursive=True) + except (psutil.NoSuchProcess, psutil.AccessDenied, PermissionError): + workers = [] + for worker in reversed(workers): + try: + worker.kill() + except psutil.NoSuchProcess: + pass + # A reaped group leader can leave workers holding stdout/stderr open. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + _ = await asyncio.wait_for(process.communicate(), timeout=1) + except TimeoutError: + pass + raise + finally: + transport.close() + return ProcessResult( + stdout=stdout.decode(errors='replace'), + stderr=stderr.decode(errors='replace'), + return_code=process.returncode if process.returncode is not None else -1, + ) diff --git a/src/scopebench_ludus/ansible/tasks.py b/src/scopebench_ludus/ansible/tasks.py new file mode 100644 index 0000000..3cf08a2 --- /dev/null +++ b/src/scopebench_ludus/ansible/tasks.py @@ -0,0 +1,52 @@ +"""Typed task construction; preserve literal strings until YAML serialization.""" + +from dataclasses import dataclass +from typing import TypedDict + +from pydantic import JsonValue + + +@dataclass(frozen=True, kw_only=True) +class AnsibleTask: + """The module call and task options used by Harbor guest operations.""" + + module: str + parameters: JsonValue + name: str | None = None + args: dict[str, JsonValue] | None = None + environment: dict[str, str] | None = None + register: str | None = None + failed_when: bool | None = None + no_log: bool | None = None + delegate_to: str | None = None + vars: dict[str, JsonValue] | None = None + loop: list[str] | None = None + + def document(self) -> dict[str, JsonValue]: + """Produce Ansible's module-key syntax without stripping !unsafe string tags.""" + result: dict[str, JsonValue] = {self.module: self.parameters} + for key, value in ( + ('name', self.name), + ('args', self.args), + ('register', self.register), + ('failed_when', self.failed_when), + ('no_log', self.no_log), + ('delegate_to', self.delegate_to), + ('vars', self.vars), + ): + if value is not None: + result[key] = value + if self.environment is not None: + result['environment'] = {key: value for key, value in self.environment.items()} + if self.loop is not None: + result['loop'] = [value for value in self.loop] + return result + + +class Play(TypedDict): + """The fixed playbook envelope used to execute a group of guest operations.""" + + name: str + hosts: str + gather_facts: bool + tasks: list[dict[str, JsonValue]] diff --git a/src/scopebench_ludus/client/__init__.py b/src/scopebench_ludus/client/__init__.py new file mode 100644 index 0000000..a5d9841 --- /dev/null +++ b/src/scopebench_ludus/client/__init__.py @@ -0,0 +1,4 @@ +"""Asynchronous Python client for the Ludus 2.x HTTP API.""" + +from .client import LudusClient as LudusClient, validate_url as validate_url +from .models import LudusAPIError as LudusAPIError, RangeStatus as RangeStatus diff --git a/src/scopebench_ludus/client/client.py b/src/scopebench_ludus/client/client.py new file mode 100644 index 0000000..9e88225 --- /dev/null +++ b/src/scopebench_ludus/client/client.py @@ -0,0 +1,239 @@ +"""Small asynchronous Ludus 2.x API client, independent of Harbor and Ansible. + +Endpoints and multipart formats follow Ludus's OpenAPI specification and CLI: +https://github.com/badsectorlabs/ludus/blob/main/docs/static/api/ludus.openapi.yaml +Range operations deliberately require an explicit range ID. +""" + +from __future__ import annotations + +import io +import re +import tarfile +from pathlib import Path +from urllib.parse import urlsplit + +import httpx +from pydantic import JsonValue, TypeAdapter, ValidationError + +from .models import AnsibleItem, LudusAPIError, RangeStatus + + +def validate_url(url: str) -> str: + """Require HTTPS without credentials, query parameters, or fragments.""" + parsed = urlsplit(url) + if ( + parsed.scheme != 'https' + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + raise ValueError('Expected an HTTPS Ludus URL without embedded credentials') + return url.rstrip('/') + + +def range_params(range_id: str) -> dict[str, str]: + """Prevent accidental requests against the user's default range.""" + if re.fullmatch(r'[A-Za-z][A-Za-z0-9_-]*', range_id) is None: + raise ValueError('An explicit Ludus range ID is required') + return {'rangeID': range_id} + + +class LudusClient: + """Use HTTP status codes and validated JSON for the required Ludus endpoints.""" + + def __init__( + self, + url: str, + api_key: str, + *, + verify: bool = True, + timeout: float = 120, + transport: httpx.AsyncBaseTransport | None = None, + ) -> None: + url = validate_url(url) + if not api_key: + raise ValueError('A Ludus API key is required') + self._timeout: float = timeout + base = url if url.endswith('/api/v2') else url + '/api/v2' + self._http: httpx.AsyncClient = httpx.AsyncClient( + base_url=base + '/', + headers={'X-API-KEY': api_key}, + verify=verify, + timeout=timeout, + transport=transport, + trust_env=False, + ) + + async def __aenter__(self) -> LudusClient: + """Open a client scope.""" + return self + + async def __aexit__(self, *_exc: object) -> None: + """Release HTTP connections on success, failure, or cancellation.""" + await self.close() + + async def close(self) -> None: + """Close controller connections even after a failed operation.""" + await self._http.aclose() + + async def request( + self, + method: str, + path: str, + *, + params: dict[str, str] | None = None, + body: dict[str, JsonValue] | None = None, + file: tuple[str, bytes, str] | None = None, + form: dict[str, str] | None = None, + timeout: float | None = None, + ) -> JsonValue: + """Make one request; never retry a potentially completed mutation blindly.""" + parsed = urlsplit(path) + if parsed.scheme or parsed.netloc or any(part == '..' for part in path.split('/')): + raise ValueError('API paths must be relative to the configured Ludus server') + path = path.lstrip('/') + if path == 'range' or path.startswith('range/'): + _ = range_params((params or {}).get('rangeID', '')) + operation = f'{method} /{path}' + try: + response = await self._http.request( + method, + path.lstrip('/'), + params=params, + json=body, + files={'file': file} if file else None, + data=form, + timeout=self._timeout if timeout is None else timeout, + ) + except httpx.RequestError: + raise LudusAPIError(operation + ' failed or timed out; check resource state') from None + if not response.is_success: + raise LudusAPIError(operation + ' failed', response.status_code) + try: + data: JsonValue = ( + TypeAdapter[JsonValue](JsonValue).validate_json(response.content) + if response.content + else {} + ) + except ValidationError: + raise LudusAPIError(operation + ' returned invalid JSON') from None + if isinstance(data, dict) and ( + data.get('error') or data.get('errors') or data.get('success') is False + ): + raise LudusAPIError(operation + ' returned an API error', response.status_code) + return data + + async def text(self, path: str, *, range_id: str | None = None) -> str: + """Read a JSON result envelope such as version or exported inventory.""" + data = await self.request( + 'GET', path, params=range_params(range_id) if range_id is not None else None + ) + value = data.get('result') if isinstance(data, dict) else None + if not isinstance(value, str): + raise LudusAPIError(f'GET {path} returned an invalid result') + return value + + async def current_user(self) -> str: + """Resolve the authenticated user without deriving identity from a credential.""" + data = await self.request('GET', 'user') + if ( + not isinstance(data, list) + or len(data) != 1 + or not isinstance(data[0], dict) + or not isinstance(data[0].get('userID'), str) + ): + raise LudusAPIError('GET user returned an invalid identity') + identity = data[0]['userID'] + assert isinstance(identity, str) + return identity + + async def create_range(self, range_id: str, name: str, description: str, user: str) -> None: + """Create and assign a new range without changing the default range.""" + _ = range_params(range_id) + _ = await self.request( + 'POST', + 'ranges/create', + body={'rangeID': range_id, 'name': name, 'description': description, 'userID': [user]}, + ) + + async def status(self, range_id: str, *, timeout: float | None = None) -> RangeStatus: + """Read the specified range and fail closed on an unexpected identity.""" + data = await self.request('GET', 'range', params=range_params(range_id), timeout=timeout) + try: + status = RangeStatus.model_validate(data) + except ValidationError: + raise LudusAPIError('GET range returned an invalid status') from None + if status.range_id != range_id: + raise LudusAPIError('GET range returned a different range identity') + return status + + async def configure_range(self, range_id: str, content: bytes) -> None: + """Upload native Ludus YAML without translating its configuration schema.""" + _ = await self.request( + 'PUT', + 'range/config', + params=range_params(range_id), + file=('range-config.yml', content, 'application/yaml'), + form={'force': 'false'}, + ) + + async def deploy(self, range_id: str) -> None: + """Deploy all VMs and native roles in the specified range.""" + _ = await self.request('POST', 'range/deploy', params=range_params(range_id), body={}) + + async def inventory(self, range_id: str) -> str: + """Export inventory for exactly one range, including connection variables.""" + return await self.text('range/ansibleinventory', range_id=range_id) + + async def abort(self, range_id: str) -> None: + """Stop a deployment before deleting its range.""" + _ = await self.request('POST', 'range/abort', params=range_params(range_id), body={}) + + async def delete_range(self, range_id: str) -> None: + """Remove the range object and its VMs, rather than only wiping its VMs.""" + _ = await self.request( + 'DELETE', 'range', params={**range_params(range_id), 'force': 'true'} + ) + + async def power_off(self, range_id: str) -> None: + """Power off every VM in the specified range.""" + _ = await self.request( + 'PUT', + 'range/poweroff', + params=range_params(range_id), + body={'machines': ['all']}, + ) + + async def upload_role(self, name: str, directory: Path) -> None: + """Upload a user-scoped role under the supplied name.""" + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode='w') as archive: + archive.add(directory, arcname=name) + _ = await self.request( + 'PUT', + 'ansible/role/fromtar', + file=(name, stream.getvalue(), 'application/x-tar'), + form={'force': 'false', 'global': 'false'}, + ) + + async def remove_role(self, name: str) -> None: + """Remove one user-scoped role.""" + _ = await self.request( + 'POST', + 'ansible/role', + body={'role': name, 'action': 'remove', 'global': False}, + ) + + async def role_exists(self, name: str) -> bool: + """Reconcile removals; Ludus can report an absent role as 409 or 500.""" + data = await self.request('GET', 'ansible') + try: + items = TypeAdapter(list[AnsibleItem]).validate_python(data) + except ValidationError: + raise LudusAPIError('GET ansible returned invalid role information') from None + return any( + item.name == name and item.type == 'role' and not item.global_scope for item in items + ) diff --git a/src/scopebench_ludus/client/models.py b/src/scopebench_ludus/client/models.py new file mode 100644 index 0000000..93dc292 --- /dev/null +++ b/src/scopebench_ludus/client/models.py @@ -0,0 +1,32 @@ +"""Validated Ludus response types and API errors.""" + +from typing import ClassVar + +from pydantic import BaseModel, ConfigDict, Field + + +class RangeStatus(BaseModel): + """Validate the identity and lifecycle fields used by the provider.""" + + model_config: ClassVar[ConfigDict] = ConfigDict(strict=True, hide_input_in_errors=True) + range_id: str = Field(alias='rangeID') + state: str = Field(alias='rangeState') + name: str + description: str + + +class LudusAPIError(RuntimeError): + """Report the operation and HTTP status without exposing response secrets.""" + + def __init__(self, operation: str, status_code: int | None = None) -> None: + self.status_code: int | None = status_code + super().__init__(f'Ludus {operation}' + (f' (HTTP {status_code})' if status_code else '')) + + +class AnsibleItem(BaseModel): + """Fields needed to distinguish a user's role from global installations.""" + + model_config: ClassVar[ConfigDict] = ConfigDict(strict=True, hide_input_in_errors=True) + name: str + type: str + global_scope: bool = Field(default=False, alias='global') diff --git a/src/scopebench_ludus/client/py.typed b/src/scopebench_ludus/client/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/scopebench_ludus/config.py b/src/scopebench_ludus/config.py new file mode 100644 index 0000000..e9124c6 --- /dev/null +++ b/src/scopebench_ludus/config.py @@ -0,0 +1,105 @@ +"""Read Ludus task assets and external controller settings.""" + +from __future__ import annotations + +import os +import re +import tomllib +from pathlib import Path, PurePosixPath +from typing import ClassVar, cast + +from pydantic import ConfigDict, Field, SecretStr + +from scopebench_contribution.runtime import runtime_files +from scopebench_contribution.schemas import Payload + + +class Interaction(Payload): + """The task's interaction tooling and external requirements.""" + + model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) + mode: str = Field(default='shell', pattern=r'^[a-z][a-z0-9-]*$') + preseeded_callback: bool = False + required_env: list[str] = Field(default_factory=list) + + +class LudusConfig(Payload): + """Settings from provider.toml: native range assets and the Harbor execution host.""" + + model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid', strict=True) + schema_version: int = Field(default=1, ge=1, le=1) + range_config: str = 'ludus/range-config.yml' + local_roles: list[str] = Field(default_factory=list) + playbook: str | None = None + host: str = Field( + default='scopebench', pattern=r'^(?:\{\{ ?range_id ?\}\}|[A-Za-z_])[A-Za-z0-9_.-]*$' + ) + interaction: Interaction = Field(default_factory=Interaction) + + def execution_host(self, range_id: str) -> str: + """Expand only Ludus's range ID token, without evaluating arbitrary Jinja.""" + return re.sub(r'\{\{ ?range_id ?\}\}', range_id, self.host) + + +def read_config(directory: Path) -> LudusConfig: + """Read and validate provider.toml and its referenced runtime files.""" + config = LudusConfig.model_validate( + cast(object, tomllib.loads((directory / 'provider.toml').read_text(encoding='utf-8'))) + ) + validate_definition_file(directory, config.range_config) + role_names: set[str] = set() + for role in config.local_roles: + validate_definition_file(directory, f'{role}/meta/main.yml') + _ = runtime_files(directory / role) + name = PurePosixPath(role).name + if not name.isidentifier() or not name.isascii() or name.lower() in role_names: + raise ValueError('Local Ludus roles must have unique identifier directory names') + role_names.add(name.lower()) + if config.playbook is not None: + validate_definition_file(directory, config.playbook) + for key in config.interaction.required_env: + if not key.isidentifier() or not key.isascii(): + raise ValueError('Interaction requirements must name environment variables') + return config + + +def validate_definition_file(directory: Path, filename: str) -> None: + """Require provider assets to resolve to ordinary files inside the definition.""" + name = PurePosixPath(filename) + if ( + name.is_absolute() + or any(part in ('', '.', '..') for part in filename.split('/')) + or '\\' in filename + or not (directory / filename).is_file() + or not (directory / filename).resolve().is_relative_to(directory.resolve()) + ): + raise ValueError('Ludus configuration must name a file inside the definition') + + +class ControllerSettings(Payload): + """Validate external controller settings once, before creating resources.""" + + url: str + api_key: SecretStr + verify: bool + deploy_timeout: float = Field(gt=0, allow_inf_nan=False) + proxy: str | None = None + + @classmethod + def from_environment(cls) -> ControllerSettings: + """Keep credentials out of authored definitions and recovery records.""" + from .client import validate_url + + verification = os.environ.get('LUDUS_VERIFY', 'true').lower() + if verification not in ('true', 'false'): + raise ValueError('LUDUS_VERIFY must be true or false') + key = os.environ.get('LUDUS_API_KEY', '') + if not key: + raise ValueError('Set LUDUS_API_KEY in the controller environment') + return cls( + url=validate_url(os.environ.get('LUDUS_URL', '')), + api_key=SecretStr(key), + verify=verification == 'true', + deploy_timeout=float(os.environ.get('LUDUS_DEPLOY_TIMEOUT_SEC', '1800')), + proxy=os.environ.get('LUDUS_WINRM_PROXY'), + ) diff --git a/src/scopebench_ludus/environment.py b/src/scopebench_ludus/environment.py new file mode 100644 index 0000000..4d3a051 --- /dev/null +++ b/src/scopebench_ludus/environment.py @@ -0,0 +1,229 @@ +"""Harbor adapter for arbitrary native Ludus ranges and Ansible execution.""" + +from __future__ import annotations + +import asyncio +import hashlib +import os +import shutil +import tempfile +from pathlib import Path +from typing import override + +from harbor.environments.base import ( # pyright: ignore[reportMissingTypeStubs] + BaseEnvironment, + ExecResult, +) +from harbor.environments.capabilities import ( # pyright: ignore[reportMissingTypeStubs] + EnvironmentCapabilities, +) +from harbor.models.task.config import TaskOS # pyright: ignore[reportMissingTypeStubs] +from harbor.models.trial.paths import EnvironmentPaths # pyright: ignore[reportMissingTypeStubs] + +from scopebench_contribution.runtime import read_reference, snapshot_runtime + +from .ansible import Guest +from .config import ControllerSettings, LudusConfig, read_config +from .range import TrialRange + + +class LudusEnvironment(BaseEnvironment): + """Create a whole range and execute on one explicitly selected inventory host.""" + + _started: bool = False + _range: TrialRange | None = None + _private: tempfile.TemporaryDirectory[str] | None = None + _guest: Guest | None = None + _config: LudusConfig | None = None + _settings: ControllerSettings + + @property + @override + def capabilities(self) -> EnvironmentCapabilities: + return EnvironmentCapabilities(windows=True) + + @property + def range_id(self) -> str | None: + """Expose the range identity for inspection and recovery.""" + return self._range.resources.range_id if self._range else None + + @property + def resource_record(self) -> Path: + """Keep each environment's recovery record separate within a shared trial.""" + identity = hashlib.sha256(self.session_id.encode()).hexdigest() + return self.trial_paths.trial_dir / 'ludus-resources' / f'{identity}.json' + + @staticmethod + @override + def type() -> str: + return 'ludus' + + @override + def _validate_definition(self) -> None: + if read_reference(self.environment_dir).backend != 'ludus': + raise ValueError('Task binding selects a different environment backend') + + @property + def config(self) -> LudusConfig: + """Return the verified task configuration.""" + if self._config is None: + raise ValueError('Ludus configuration has not been loaded') + return self._config + + @property + def guest(self) -> Guest: + """The selected VM used for Harbor commands and file transfers.""" + if self._guest is None: + raise ValueError('Ludus execution host is not configured') + return self._guest + + @classmethod + @override + def preflight(cls) -> None: + for executable in ('ansible-playbook', 'ansible-inventory'): + if not shutil.which(executable): + raise ValueError(f'Missing {executable}; run uv sync --locked --group ludus') + + def configure(self, definition: Path) -> None: + """Validate task and controller settings before allocating resources.""" + self._config = read_config(definition) + self.preflight() + self._settings = ControllerSettings.from_environment() + for variable in self.config.interaction.required_env: + if not os.environ.get(variable): + raise ValueError(f'Missing controller environment variable: {variable}') + # Ludus owns topology validation. Interaction metadata describes the task's + # tooling; it does not restrict how many VMs or operating systems may exist. + + async def _provision(self, definition: Path) -> None: + if self._range and (self._range.resources.state != 'absent' or self._range.resources.roles): + raise ValueError('Clean up the previous trial resources before restarting') + self._range = TrialRange( + self._settings, + self.config, + self.session_id, + self.resource_record, + self.logger, + ) + self._private = tempfile.TemporaryDirectory(prefix='scopebench-ludus-') + directory = Path(self._private.name) + self._guest = Guest( + directory, + self.config.execution_host(self._range.resources.range_id), + self.os == TaskOS.WINDOWS, + self._settings.proxy, + ) + inventory = directory / 'inventory' + _ = inventory.write_text( + f'[{self.guest.ansible.host}]\nvalidation.invalid\n', encoding='utf-8' + ) + if self.config.playbook: + await self.guest.ansible.playbook(definition / self.config.playbook, syntax_check=True) + _ = inventory.write_text(await self._range.deploy(definition, directory), encoding='utf-8') + inventory.chmod(0o600) + await self.guest.select_host() + paths = EnvironmentPaths.for_os(self.os) + await self.guest.files.create_directories( + [ + str(paths.agent_dir), + str(paths.verifier_dir), + str(paths.artifacts_dir), + *self._mount_targets(), + ] + ) + if self.config.playbook: + await self.guest.ansible.playbook(definition / self.config.playbook) + + async def _cleanup(self, *, delete: bool) -> None: + if self._range is None: + return + + async def cleanup() -> None: + assert self._range is not None + try: + await self._range.cleanup(delete=delete) + finally: + if self._private is not None: + self._private.cleanup() + self._private = None + self._guest = None + + task = asyncio.create_task(cleanup()) + try: + await asyncio.shield(task) + except asyncio.CancelledError: + await task + raise + except Exception: + self.logger.error('Retry Ludus cleanup with --resources %s', self.resource_record) + raise + + @override + async def start(self, force_build: bool) -> None: + # Each trial starts from templates; force_build does not reuse VMs. + if self._started: + raise ValueError('Environment instance already started') + _, definition = snapshot_runtime( + self.environment_dir, + self.trial_paths.trial_dir + / 'runtime-definitions' + / hashlib.sha256(self.session_id.encode()).hexdigest(), + ) + self.configure(definition) + try: + await self._provision(definition) + except BaseException as failure: + try: + await self._cleanup(delete=True) + except BaseException as cleanup: + failure.add_note( + f'Ludus cleanup also failed ({type(cleanup).__name__}); inspect trial resources' + ) + raise + self._started = True + + @override + async def stop(self, delete: bool) -> None: + # Failed starts and retained ranges can still have resources to release. + await self._cleanup(delete=delete) + self._started = False + + @override + async def exec( + self, + command: str, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout_sec: int | None = None, + user: str | int | None = None, + ) -> ExecResult: + result = await self.guest.exec( + command, + cwd=cwd, + env=self._merge_env(env), + timeout_sec=timeout_sec, + user=self._resolve_user(user), + ) + callback = self._output_callback() + if callback is not None: + await callback(result.stdout or '', 'stdout') + await callback(result.stderr or '', 'stderr') + return ExecResult( + stdout=result.stdout, stderr=result.stderr, return_code=result.return_code + ) + + @override + async def upload_file(self, source_path: Path | str, target_path: str) -> None: + await self.guest.files.upload_file(source_path, target_path) + + @override + async def upload_dir(self, source_dir: Path | str, target_dir: str) -> None: + await self.guest.files.upload_dir(source_dir, target_dir) + + @override + async def download_file(self, source_path: str, target_path: Path | str) -> None: + await self.guest.files.download_file(source_path, target_path) + + @override + async def download_dir(self, source_dir: str, target_dir: Path | str) -> None: + await self.guest.files.download_dir(source_dir, target_dir) diff --git a/src/scopebench_ludus/range.py b/src/scopebench_ludus/range.py new file mode 100644 index 0000000..68dede0 --- /dev/null +++ b/src/scopebench_ludus/range.py @@ -0,0 +1,103 @@ +"""Create, deploy and release the complete range owned by one Harbor trial.""" + +import asyncio +import logging +import time +from pathlib import Path +from uuid import uuid4 + +from .client import LudusAPIError, LudusClient +from .config import ControllerSettings, LudusConfig +from .resources import CLEANUP_TIMEOUT_SEC, LudusResources, cleanup_resources +from .roles import upload_roles + + +class TrialRange: + """Own one range and its uploaded roles; persist intent before API writes.""" + + def __init__( + self, + settings: ControllerSettings, + config: LudusConfig, + name: str, + record: Path, + logger: logging.Logger, + ) -> None: + self.settings: ControllerSettings = settings + self.config: LudusConfig = config + self.record: Path = record + self.logger: logging.Logger = logger + self.resources: LudusResources = LudusResources( + url=settings.url, + verify=settings.verify, + range_id='SB' + uuid4().hex, + name=name, + ) + self.resources.save(record) + + def _client(self, *, timeout: float = 120) -> LudusClient: + return LudusClient( + self.settings.url, + self.settings.api_key.get_secret_value(), + verify=self.settings.verify, + timeout=timeout, + ) + + async def _wait(self, client: LudusClient, resources: LudusResources) -> None: + deadline = time.monotonic() + self.settings.deploy_timeout + previous: str | None = None + async with asyncio.timeout(self.settings.deploy_timeout): + while True: + status = await client.status( + resources.range_id, timeout=min(120, max(0.001, deadline - time.monotonic())) + ) + if status.name != resources.name or status.description != resources.description: + raise RuntimeError('Ludus range ownership marker differs') + if status.state != previous: + self.logger.info('Ludus range %s: %s', resources.range_id, status.state) + previous = status.state + if status.state == 'SUCCESS': + return + if status.state in ('ERROR', 'ABORTED', 'DESTROYED', 'DESTROYING'): + raise RuntimeError(f'Ludus range {resources.range_id}: {status.state}') + await asyncio.sleep(min(5, max(0, deadline - time.monotonic()))) + + async def deploy(self, definition: Path, directory: Path) -> str: + """Deploy native YAML and roles, then return the range's Ansible inventory.""" + resources = self.resources + client = self._client() + try: + _ = await client.text('') + user = await client.current_user() + resources.state = 'pending' + resources.save(self.record) + try: + await client.create_range( + resources.range_id, resources.name, resources.description, user + ) + except LudusAPIError as error: + if error.status_code is not None and 400 <= error.status_code < 500: + resources.state = 'absent' + resources.save(self.record) + raise + resources.state = 'owned' + resources.save(self.record) + self.logger.info('Created Ludus range %s for %s', resources.range_id, resources.name) + config = await upload_roles( + client, definition, self.config, directory, resources, self.record + ) + await client.configure_range(resources.range_id, config) + await client.deploy(resources.range_id) + await self._wait(client, resources) + return await client.inventory(resources.range_id) + finally: + await client.close() + + async def cleanup(self, *, delete: bool) -> None: + """Reconcile recorded resources within one overall deletion deadline.""" + async with self._client(timeout=CLEANUP_TIMEOUT_SEC) as client: + try: + async with asyncio.timeout(CLEANUP_TIMEOUT_SEC): + await cleanup_resources(client, self.resources, self.record, delete=delete) + finally: + self.resources.save(self.record) diff --git a/src/scopebench_ludus/resources.py b/src/scopebench_ludus/resources.py new file mode 100644 index 0000000..2006573 --- /dev/null +++ b/src/scopebench_ludus/resources.py @@ -0,0 +1,160 @@ +"""Persist trial-owned resource identities and retry incomplete cleanup.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from typing import ClassVar, Literal, Self + +from pydantic import ConfigDict, Field, model_validator + +from scopebench_contribution.schemas import Payload + +from .client import LudusAPIError, LudusClient, validate_url + +CLEANUP_TIMEOUT_SEC = 120 + + +class LudusResources(Payload): + """A credential-free recovery record; pending writes survive controller failure.""" + + model_config: ClassVar[ConfigDict] = ConfigDict( + extra='forbid', strict=True, hide_input_in_errors=True + ) + url: str + verify: bool = True + range_id: str = Field(pattern=r'^SB[0-9a-f]{32}$') + name: str + state: Literal['absent', 'pending', 'owned'] = 'absent' + roles: list[str] = Field(default_factory=list) + + @model_validator(mode='after') + def validate_record(self) -> Self: + """Restrict cleanup to the record's trial namespace.""" + _ = validate_url(self.url) + prefix = f'scopebench_{self.range_id}_'.lower() + if any(not role.startswith(prefix) or not role.isidentifier() for role in self.roles): + raise ValueError('Recovery role names must belong to this trial') + return self + + @property + def description(self) -> str: + """Use the complete random trial ID as the server-side ownership marker.""" + return 'ScopeBench trial ' + self.range_id + + def save(self, path: Path) -> None: + """Atomically save resource state without inventory or API credentials.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix('.tmp') + _ = temporary.write_text(self.model_dump_json(indent=2) + '\n', encoding='utf-8') + temporary.chmod(0o600) + _ = temporary.replace(path) + + +def _range_failure(resources: LudusResources, error: LudusAPIError) -> list[str]: + """Reconcile a confirmed missing range while retaining uncertain create intent.""" + if error.status_code == 404 and resources.state == 'owned': + resources.state = 'absent' + return [] + return [f'range {resources.range_id}: {error}'] + + +async def _delete_range( + client: LudusClient, resources: LudusResources, *, deploying: bool +) -> list[str]: + """Abort an active deployment and delete its range, reconciling an uncertain abort.""" + failures: list[str] = [] + if deploying: + try: + await client.abort(resources.range_id) + except LudusAPIError as error: + failures.append(f'abort {resources.range_id}: {error}') + try: + await client.delete_range(resources.range_id) + except LudusAPIError as error: + return failures + _range_failure(resources, error) + resources.state = 'absent' + # Successful deletion also resolves a failed or uncertain abort. + return [] + + +async def _cleanup_range( + client: LudusClient, resources: LudusResources, record: Path, *, delete: bool +) -> list[str]: + """Verify ownership before deleting or powering off the recorded range.""" + if resources.state == 'absent': + return [] + try: + status = await client.status(resources.range_id) + except LudusAPIError as error: + return _range_failure(resources, error) + if status.name != resources.name or status.description != resources.description: + return ['range ownership ' + resources.range_id] + resources.state = 'owned' + resources.save(record) + if delete: + return await _delete_range(client, resources, deploying=status.state == 'DEPLOYING') + try: + await client.power_off(resources.range_id) + except LudusAPIError as error: + return _range_failure(resources, error) + return [] + + +async def _remove_role(client: LudusClient, role: str) -> str | None: + """Treat an upload that is already absent as successfully removed.""" + try: + await client.remove_role(role) + except LudusAPIError as error: + try: + if not await client.role_exists(role): + return None + except LudusAPIError: + pass + return f'role {role}: {error}' + return None + + +async def _cleanup_roles(client: LudusClient, resources: LudusResources, record: Path) -> list[str]: + """Attempt every recorded role and persist each successful removal.""" + failures: list[str] = [] + for role in list(resources.roles): + failure = await _remove_role(client, role) + if failure is not None: + failures.append(failure) + continue + resources.roles.remove(role) + resources.save(record) + return failures + + +async def cleanup_resources( + client: LudusClient, resources: LudusResources, record: Path, *, delete: bool +) -> None: + """Clean up the range and roles, retaining unresolved identities for recovery.""" + try: + failures = await _cleanup_range(client, resources, record, delete=delete) + finally: + resources.save(record) + if delete: + failures.extend(await _cleanup_roles(client, resources, record)) + if failures: + raise RuntimeError('Ludus cleanup incomplete: ' + ', '.join(failures) + f'; retry {record}') + + +async def recover(record: Path) -> None: + """Delete only recorded trial resources using the current controller credential.""" + resources = LudusResources.model_validate_json(record.read_bytes()) + client = LudusClient( + resources.url, + os.environ.get('LUDUS_API_KEY', ''), + verify=resources.verify, + timeout=CLEANUP_TIMEOUT_SEC, + ) + try: + async with asyncio.timeout(CLEANUP_TIMEOUT_SEC): + await cleanup_resources(client, resources, record, delete=True) + finally: + resources.save(record) + await client.close() diff --git a/src/scopebench_ludus/roles.py b/src/scopebench_ludus/roles.py new file mode 100644 index 0000000..37dc990 --- /dev/null +++ b/src/scopebench_ludus/roles.py @@ -0,0 +1,74 @@ +"""Namespace bundled roles while preserving native Ludus range settings.""" + +import shutil +from pathlib import Path + +import yaml +from pydantic import JsonValue, TypeAdapter + +from .client import LudusClient +from .config import LudusConfig +from .resources import LudusResources + + +def remap_roles(roles: JsonValue, aliases: dict[str, str]) -> None: + """Point VM and dependency references at the trial-specific uploaded role names.""" + if not isinstance(roles, list): + raise ValueError('Ludus VM roles must be a list') + for index, role in enumerate(roles): + if isinstance(role, str): + roles[index] = aliases.get(role, role) + elif isinstance(role, dict): + for key in ('name', 'role'): + name = role.get(key) + if isinstance(name, str): + role[key] = aliases.get(name, name) + dependencies = role.get('depends_on', []) + if isinstance(dependencies, list): + for dependency in dependencies: + if isinstance(dependency, dict): + name = dependency.get('role') + if isinstance(name, str): + dependency['role'] = aliases.get(name, name) + + +async def upload_roles( + client: LudusClient, + definition: Path, + config: LudusConfig, + directory: Path, + resources: LudusResources, + record: Path, +) -> bytes: + """Upload bundled roles under trial names and return native range YAML.""" + source = definition / config.range_config + if not config.local_roles: + return source.read_bytes() + aliases = { + Path(role).name: f'scopebench_{resources.range_id}_{Path(role).name}'.lower() + for role in config.local_roles + } + for relative in config.local_roles: + role = definition / relative + name = aliases[role.name] + target = directory / 'roles' / name + _ = shutil.copytree(role, target) + meta = target / 'meta/main.yml' + metadata = TypeAdapter(dict[str, JsonValue]).validate_python( + yaml.safe_load(meta.read_text()) + ) + if 'dependencies' in metadata: + remap_roles(metadata['dependencies'], aliases) + _ = meta.write_text(yaml.safe_dump(metadata, sort_keys=False), encoding='utf-8') + resources.roles.append(name) + resources.save(record) + await client.upload_role(name, target) + native = TypeAdapter(dict[str, JsonValue]).validate_python(yaml.safe_load(source.read_text())) + hosts = native.get('ludus') + if not isinstance(hosts, list): + raise ValueError('Native Ludus configuration must contain a ludus VM list') + for host in hosts: + if not isinstance(host, dict): + raise ValueError('Native Ludus VM definitions must be objects') + remap_roles(host.get('roles', []), aliases) + return yaml.safe_dump(native, sort_keys=False).encode() diff --git a/src/scopebench_environments/ruff.toml b/src/scopebench_ludus/ruff.toml similarity index 100% rename from src/scopebench_environments/ruff.toml rename to src/scopebench_ludus/ruff.toml diff --git a/src/scopebench_ludus/setup.py b/src/scopebench_ludus/setup.py new file mode 100644 index 0000000..f397755 --- /dev/null +++ b/src/scopebench_ludus/setup.py @@ -0,0 +1,38 @@ +"""Install the Ansible Windows modules used for guest commands and files.""" + +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +def setup_ludus() -> None: + """Install the pinned controller collection once; never change a Ludus server.""" + for executable in ('ansible-playbook', 'ansible-inventory', 'ansible-galaxy'): + if not shutil.which(executable): + raise ValueError('Run uv sync --locked --group ludus before setup') + destination = Path(sys.prefix) / 'ansible/collections' + environment = dict(os.environ) + environment['ANSIBLE_COLLECTIONS_PATH'] = str(destination) + _ = environment.pop('LUDUS_API_KEY', None) + _ = environment.pop('LUDUS_JWT_TOKEN', None) + with tempfile.TemporaryDirectory(prefix='scopebench-ansible-setup-') as directory: + environment['ANSIBLE_LOCAL_TEMP'] = directory + environment['ANSIBLE_GALAXY_CACHE_DIR'] = directory + result = subprocess.run( + [ + 'ansible-galaxy', + 'collection', + 'install', + 'ansible.windows:3.2.0', + '-p', + str(destination), + ], + env=environment, + timeout=300, + check=False, + ) + if result.returncode: + raise RuntimeError('Ansible controller collection installation failed') diff --git a/tests/ludus/__init__.py b/tests/ludus/__init__.py new file mode 100644 index 0000000..a0732dd --- /dev/null +++ b/tests/ludus/__init__.py @@ -0,0 +1 @@ +"""Ludus environment, guest and ownership contract tests.""" diff --git a/tests/ludus/conftest.py b/tests/ludus/conftest.py new file mode 100644 index 0000000..ae8ed30 --- /dev/null +++ b/tests/ludus/conftest.py @@ -0,0 +1,54 @@ +"""Exercise generic range, HTTP ownership, and Ansible transport contracts.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest +from harbor.environments.factory import ( # pyright: ignore[reportMissingTypeStubs] + EnvironmentFactory, +) +from harbor.models.task.config import ( # pyright: ignore[reportMissingTypeStubs] + EnvironmentConfig, + TaskOS, +) +from harbor.models.trial.paths import TrialPaths # pyright: ignore[reportMissingTypeStubs] + +from scopebench_ludus import environment as runner, range as ranges, resources as ludus_resources +from scopebench_ludus.ansible import guest, playbook +from tests.ludus.controller import Controller +from tests.ludus_helpers import IMPORT_PATH, prepare_environment + + +@pytest.fixture +def controller(monkeypatch: pytest.MonkeyPatch) -> Controller: + fake = Controller() + monkeypatch.setattr(playbook, 'run_process', fake.run) + monkeypatch.setattr(guest, 'run_process', fake.run) + monkeypatch.setattr(ranges, 'LudusClient', fake.client) + monkeypatch.setattr(ludus_resources, 'LudusClient', fake.client) + + def executable(_name: str) -> str: + return '/fixture/bin' + + monkeypatch.setattr(shutil, 'which', executable) + monkeypatch.setenv('LUDUS_URL', 'https://192.0.2.1:8080') + monkeypatch.setenv('LUDUS_API_KEY', 'controller-only-secret') + return fake + + +@pytest.fixture +def environment(tmp_path: Path, controller: Controller) -> runner.LudusEnvironment: + _ = controller + task = prepare_environment(tmp_path) + loaded = EnvironmentFactory.create_environment_from_import_path( # pyright: ignore[reportUnknownMemberType] + import_path=IMPORT_PATH, + environment_dir=task, + environment_name='smoke', + session_id='fixture-harbor-session', + trial_paths=TrialPaths(tmp_path / 'trial'), + task_env_config=EnvironmentConfig(os=TaskOS.WINDOWS), + ) + assert isinstance(loaded, runner.LudusEnvironment) + return loaded diff --git a/tests/ludus/controller.py b/tests/ludus/controller.py new file mode 100644 index 0000000..9914c61 --- /dev/null +++ b/tests/ludus/controller.py @@ -0,0 +1,244 @@ +"""Exercise generic range, HTTP ownership, and Ansible transport contracts.""" + +from __future__ import annotations + +import asyncio +import io +import json +import tarfile +from email.message import Message +from email.parser import BytesParser +from pathlib import Path +from typing import cast, final + +import httpx +import yaml + +from scopebench_ludus.ansible.process import ProcessResult +from scopebench_ludus.client import LudusClient + + +class TransportLoader(yaml.SafeLoader): + """Read transport literals without evaluating expressions.""" + + +def load_literal(loader: yaml.SafeLoader, node: yaml.ScalarNode) -> str: + return loader.construct_scalar(node) + + +def uploaded_file(request: httpx.Request) -> bytes: + """Decode the actual HTTP multipart upload independently of the client.""" + message = BytesParser().parsebytes( + f'Content-Type: {request.headers["content-type"]}\r\n\r\n'.encode() + request.content + ) + parts = cast(list[Message], message.get_payload()) + for part in parts: + if part.get_param('name', header='content-disposition') == 'file': + return cast(bytes, part.get_payload(decode=True)) + raise AssertionError('No multipart file') + + +TransportLoader.add_constructor('!unsafe', load_literal) + + +@final +class Controller: + """Mock the HTTP server and Ansible results at their external boundaries.""" + + def __init__(self) -> None: + self.calls: list[httpx.Request] = [] + self.events: list[str] = [] + self.processes: list[list[str]] = [] + self.plays: list[list[dict[str, object]]] = [] + self.environments: list[dict[str, str]] = [] + self.raw_playbooks: list[str] = [] + self.state = 'SUCCESS' + self.fail_create = False + self.lost_create = False + self.unobserved_create = False + self.fail_delete = False + self.fail_abort = False + self.fail_role_lookup = False + self.delete_response_delay = 0.0 + self.fail_play = False + self.wrong_range = False + self.fail_role = False + self.fail_role_remove = False + self.cancel_deploy = False + self.windows = True + self.selected_hosts = ['executor'] + self.ranges: dict[str, dict[str, str]] = {} + self.range_configs: list[dict[str, object]] = [] + self.role_tasks: list[str] = [] + self.role_metadata: list[str] = [] + self.removed_roles: list[str] = [] + self.installed_roles: set[str] = set() + self.entries: list[dict[str, object]] = [] + + def request(self, request: httpx.Request) -> httpx.Response: + self.calls.append(request) + path = request.url.path.removeprefix('/api/v2/') + self.events.append(request.method + ' ' + path) + assert request.headers['X-API-KEY'] == 'controller-only-secret' + if path == '': + return httpx.Response(200, json={'result': 'Ludus Server 2.3.5'}) + if path == 'user': + return httpx.Response(200, json=[{'userID': 'fixture'}]) + if path == 'ranges/create': + return self.create_range(request) + elif path == 'range' and request.method == 'GET': + return self.status(request) + elif path == 'range' and request.method == 'DELETE': + assert request.url.params['force'] == 'true' + if self.fail_delete: + return httpx.Response(500, json={'error': 'fixture secret'}) + del self.ranges[request.url.params['rangeID']] + # Model a completed synchronous deletion whose response arrives late. + timeouts = cast(dict[str, float | None], request.extensions['timeout']) + read_timeout = timeouts['read'] + if read_timeout is not None and read_timeout < self.delete_response_delay: + raise httpx.ReadTimeout('Delete response did not arrive in time', request=request) + elif path == 'range/config': + self.range_configs.append( + cast(dict[str, object], yaml.safe_load(uploaded_file(request))) + ) + elif path == 'range/deploy' and self.cancel_deploy: + raise asyncio.CancelledError + elif path == 'ansible/role/fromtar': + return self.upload_role(request) + elif path == 'range/abort' and self.fail_abort: + return httpx.Response(500, json={'error': 'fixture secret'}) + elif path == 'ansible': + return self.list_roles() + elif path == 'ansible/role': + return self.remove_role(request) + elif path == 'range/ansibleinventory': + connection = 'winrm' if self.windows else 'ssh' + return httpx.Response( + 200, + json={ + 'result': f'all:\n hosts:\n executor:\n ansible_connection: {connection}\n ansible_password: fixture\n' + }, + ) + return httpx.Response(201, json={'result': 'ok'}) + + def list_roles(self) -> httpx.Response: + if self.fail_role_lookup: + return httpx.Response(500, json={'error': 'fixture secret'}) + return httpx.Response( + 200, + json=[{'name': name, 'type': 'role', 'global': False} for name in self.installed_roles], + ) + + def remove_role(self, request: httpx.Request) -> httpx.Response: + data = cast(dict[str, str], json.loads(request.content)) + assert data['action'] == 'remove' + if self.fail_role_remove: + return httpx.Response(500, json={'error': 'fixture secret'}) + if data['role'] not in self.installed_roles: + return httpx.Response(409, json={'error': 'role not installed'}) + self.removed_roles.append(data['role']) + self.installed_roles.discard(data['role']) + return httpx.Response(201, json={'result': 'removed'}) + + def status(self, request: httpx.Request) -> httpx.Response: + identity = request.url.params['rangeID'] + if identity not in self.ranges: + return httpx.Response(404, json={'error': 'not found'}) + data = {**self.ranges[identity], 'rangeState': self.state} + if self.wrong_range: + data['rangeID'] = 'unrelated' + return httpx.Response(200, json=data) + + def create_range(self, request: httpx.Request) -> httpx.Response: + data = cast(dict[str, str], json.loads(request.content)) + assert data['userID'] == ['fixture'] + if self.fail_create: + return httpx.Response(409, json={'error': 'fixture secret'}) + if not self.unobserved_create: + self.ranges[data['rangeID']] = { + 'rangeID': data['rangeID'], + 'name': data['name'], + 'description': data['description'], + } + if self.lost_create or self.unobserved_create: + raise httpx.ReadTimeout('fixture secret', request=request) + return httpx.Response(201, json={'result': 'ok'}) + + def upload_role(self, request: httpx.Request) -> httpx.Response: + with tarfile.open(fileobj=io.BytesIO(uploaded_file(request))) as archive: + self.installed_roles.add(archive.getmembers()[0].name.split('/')[0]) + tasks = next( + member for member in archive.getmembers() if member.name.endswith('/tasks/main.yml') + ) + content = archive.extractfile(tasks) + assert content is not None + self.role_tasks.append(content.read().decode()) + meta = next( + member for member in archive.getmembers() if member.name.endswith('/meta/main.yml') + ) + content = archive.extractfile(meta) + assert content is not None + self.role_metadata.append(content.read().decode()) + if self.fail_role: + return httpx.Response(500, json={'error': 'fixture secret'}) + return httpx.Response(201, json={'result': 'ok'}) + + def client(self, url: str, api_key: str, *, verify: bool, timeout: float = 120) -> LudusClient: + return LudusClient( + url, + api_key, + verify=verify, + timeout=timeout, + transport=httpx.MockTransport(self.request), + ) + + async def run( + self, + command: list[str], + *, + timeout: float, + env: dict[str, str] | None = None, + ) -> ProcessResult: + _ = timeout + self.processes.append(command) + self.environments.append(env or {}) + if command[0] == 'ansible-inventory': + variables = {'executor': {'ansible_connection': 'winrm' if self.windows else 'ssh'}} + return ProcessResult( + return_code=0, + stderr='', + stdout=json.dumps( + { + '_meta': {'hostvars': variables}, + 'scopebench': {'hosts': self.selected_hosts}, + } + ), + ) + self.events.append('transport') + raw = Path(command[-1]).read_text(encoding='utf-8') + self.raw_playbooks.append(raw) + if self.fail_play and '--syntax-check' not in command: + return ProcessResult(stdout='', stderr='fixture secret', return_code=2) + plays = cast(list[dict[str, object]], yaml.load(raw, Loader=TransportLoader)) + tasks = cast(list[dict[str, object]], plays[0]['tasks']) + self.plays.append(tasks) + for task in tasks: + if 'ansible.builtin.copy' in task and 'delegate_to' in task: + copy = cast(dict[str, str], task['ansible.builtin.copy']) + payload: object = ( + self.entries + if 'scopebench_files.files' in copy['content'] + else { + 'rc': 7, + 'stdout': 'fixture stdout\r\n', + 'stderr': 'fixture stderr', + } + ) + _ = Path(copy['dest']).write_text(json.dumps(payload), encoding='utf-8') + return ProcessResult(stdout='PLAY RECAP', stderr='', return_code=0) + + @property + def inventory(self) -> Path: + command = next(call for call in self.processes if call[0] == 'ansible-inventory') + return Path(command[command.index('-i') + 1]) diff --git a/tests/ludus/test_ansible_local.py b/tests/ludus/test_ansible_local.py new file mode 100644 index 0000000..2af0976 --- /dev/null +++ b/tests/ludus/test_ansible_local.py @@ -0,0 +1,63 @@ +"""Exercise Linux command and binary transfer semantics using real local Ansible.""" + +import asyncio +import getpass +import os +import sys +from pathlib import Path + +import pytest + +from scopebench_ludus.ansible import Guest + + +def test_local_ansible_command_and_binary_directory_round_trips( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + directory = tmp_path / 'controller' + directory.mkdir() + _ = (directory / 'inventory').write_text( + f'[scopebench]\nlocalhost ansible_connection=local ansible_become=false ansible_become_user=nobody ansible_python_interpreter={sys.executable} ansible_remote_tmp={tmp_path / "modules"}\n' + ) + transport = Guest(directory, 'scopebench', False, None) + remote = tmp_path / 'remote' + source = tmp_path / 'source' + (source / 'nested').mkdir(parents=True) + (source / 'empty').mkdir() + payload = b'scopebench\x00\xff\r\n{{ literal }}' + _ = (source / 'nested/proof.bin').write_bytes(payload) + + async def exercise() -> None: + await transport.select_host() + await transport.files.create_directories([str(remote)]) + result = await transport.exec( + 'printf "%s\\n" "$SB_PROOF"; pwd; exit 7', + cwd=str(remote), + env={'SB_PROOF': '{{ literal }}'}, + timeout_sec=30, + user=None, + ) + assert result.return_code == 7 + assert (result.stdout or '').splitlines() == ['{{ literal }}', str(remote.resolve())] + identity = await transport.exec( + 'whoami', cwd=None, env=None, timeout_sec=30, user=getpass.getuser() + ) + assert (identity.stdout or '').strip() == getpass.getuser() + await transport.files.upload_file(source / 'nested/proof.bin', str(remote / 'file.bin')) + await transport.files.download_file(str(remote / 'file.bin'), tmp_path / 'file.bin') + assert (tmp_path / 'file.bin').read_bytes() == payload + await transport.files.upload_dir(source, str(remote / 'tree')) + await transport.files.download_dir(str(remote / 'tree'), tmp_path / 'download') + assert (tmp_path / 'download/nested/proof.bin').read_bytes() == payload + assert (tmp_path / 'download/empty').is_dir() + # Ordinary symlinks cannot turn a directory download into exfiltration + # outside its requested remote tree. + (remote / 'tree/link').symlink_to(source / 'nested/proof.bin') + with pytest.raises(ValueError, match='symbolic links'): + await transport.files.download_dir(str(remote / 'tree'), tmp_path / 'unsafe-download') + + # Command lookup uses this test environment's Ansible installation. + monkeypatch.setenv( + 'PATH', str(Path(sys.executable).parent) + os.pathsep + os.environ.get('PATH', '') + ) + asyncio.run(exercise()) diff --git a/tests/ludus/test_cleanup.py b/tests/ludus/test_cleanup.py new file mode 100644 index 0000000..133fc0e --- /dev/null +++ b/tests/ludus/test_cleanup.py @@ -0,0 +1,291 @@ +"""Exercise generic range, HTTP ownership, and Ansible transport contracts.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +from harbor.models.task.config import ( # pyright: ignore[reportMissingTypeStubs] + EnvironmentConfig, + TaskOS, +) +from harbor.models.trial.paths import TrialPaths # pyright: ignore[reportMissingTypeStubs] + +from scopebench_contribution.runtime import read_reference, runtime_digest +from scopebench_ludus import environment as runner, resources as ludus_resources +from scopebench_ludus.resources import LudusResources +from tests.ludus.controller import Controller + + +@pytest.mark.parametrize('resource', ['range', 'role']) +def test_cleanup_attempts_other_resources_and_can_be_retried_from_disk( + environment: runner.LudusEnvironment, + controller: Controller, + resource: str, +) -> None: + asyncio.run(environment.start(force_build=False)) + inventory = controller.inventory + controller.fail_delete = resource == 'range' + controller.fail_role_remove = resource == 'role' + with pytest.raises(RuntimeError, match='cleanup incomplete') as failure: + asyncio.run(environment.stop(delete=True)) + assert 'HTTP 500' in str(failure.value) + assert 'fixture secret' not in str(failure.value) + assert not inventory.exists() + assert bool(controller.ranges) == (resource == 'range') + assert bool(controller.removed_roles) == (resource != 'role') + controller.fail_delete = controller.fail_role_remove = False + asyncio.run(ludus_resources.recover(environment.resource_record)) + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert not controller.ranges + assert record.state == 'absent' + assert not record.roles + + +@pytest.mark.parametrize('recovery', [False, True]) +def test_slow_synchronous_delete_completes_before_role_cleanup( + environment: runner.LudusEnvironment, + controller: Controller, + recovery: bool, +) -> None: + asyncio.run(environment.start(force_build=False)) + # A four-VM live range exceeded the old 15-second request timeout. + controller.delete_response_delay = 20 + if recovery: + asyncio.run(ludus_resources.recover(environment.resource_record)) + else: + asyncio.run(environment.stop(delete=True)) + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == 'absent' + assert not record.roles + assert not controller.ranges + assert not controller.installed_roles + asyncio.run(environment.stop(delete=True)) + + +def test_conflict_never_deletes_a_range_it_did_not_create( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + controller.fail_create = True + with pytest.raises(RuntimeError, match='HTTP 409'): + asyncio.run(environment.start(force_build=False)) + assert 'DELETE range' not in controller.events + + +def test_lost_create_response_is_reconciled_by_ownership_marker( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + controller.lost_create = True + with pytest.raises(RuntimeError, match='timed out'): + asyncio.run(environment.start(force_build=False)) + assert 'DELETE range' in controller.events + assert not controller.ranges + + +def test_unobserved_create_preserves_recovery_intent( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + controller.unobserved_create = True + with pytest.raises(RuntimeError) as failure: + asyncio.run(environment.start(force_build=False)) + assert failure.value.__notes__ + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == 'pending' + controller.ranges[record.range_id] = { + 'rangeID': record.range_id, + 'name': record.name, + 'description': record.description, + } + asyncio.run(ludus_resources.recover(environment.resource_record)) + assert not controller.ranges + + +@pytest.mark.parametrize('wrong_identity', [False, True]) +def test_cleanup_refuses_a_different_identity_or_ownership_marker( + environment: runner.LudusEnvironment, + controller: Controller, + wrong_identity: bool, +) -> None: + asyncio.run(environment.start(force_build=False)) + assert environment.range_id is not None + if wrong_identity: + controller.wrong_range = True + else: + controller.ranges[environment.range_id]['description'] = 'another owner' + with pytest.raises(RuntimeError, match='cleanup incomplete'): + asyncio.run(environment.stop(delete=True)) + assert 'DELETE range' not in controller.events + assert controller.removed_roles + + +def test_keep_preserves_roles_and_allows_later_delete( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + asyncio.run(environment.start(force_build=False)) + asyncio.run(environment.stop(delete=False)) + assert 'PUT range/poweroff' in controller.events + assert controller.ranges + assert not controller.removed_roles + request = next(item for item in controller.calls if item.url.path.endswith('/range/poweroff')) + assert json.loads(request.content) == {'machines': ['all']} + asyncio.run(environment.stop(delete=True)) + assert not controller.ranges + assert controller.removed_roles + + +def test_cancellation_cleans_resources_and_local_secrets( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + controller.cancel_deploy = True + with pytest.raises(asyncio.CancelledError): + asyncio.run(environment.start(force_build=False)) + assert not controller.ranges + assert controller.removed_roles + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == 'absent' + assert not record.roles + + +def test_absent_pending_role_is_confirmed_and_removed_from_recovery_record( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + asyncio.run(environment.start(force_build=False)) + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + record.roles.append(f'scopebench_{record.range_id}_never_uploaded'.lower()) + record.save(environment.resource_record) + asyncio.run(ludus_resources.recover(environment.resource_record)) + recovered = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert not recovered.roles + assert recovered.state == 'absent' + assert 'GET ansible' in controller.events + asyncio.run(environment.stop(delete=True)) + + +@pytest.mark.parametrize('shared_trial', [False, True]) +def test_concurrent_environments_own_separate_ranges_roles_and_recovery_files( + environment: runner.LudusEnvironment, + controller: Controller, + tmp_path: Path, + shared_trial: bool, +) -> None: + second = runner.LudusEnvironment( + environment_dir=environment.environment_dir, + environment_name='second', + session_id='second-session', + trial_paths=environment.trial_paths + if shared_trial + else TrialPaths(tmp_path / 'second-trial'), + task_env_config=EnvironmentConfig(os=TaskOS.WINDOWS), + ) + + async def exercise() -> None: + _ = await asyncio.gather( + environment.start(force_build=False), second.start(force_build=False) + ) + assert environment.range_id != second.range_id + assert environment.resource_record != second.resource_record + assert len(controller.installed_roles) == 2 + directories = { + path + for trial in {environment.trial_paths.trial_dir, second.trial_paths.trial_dir} + for path in (trial / 'runtime-definitions').iterdir() + } + assert len(directories) == 2 + assert {runtime_digest(path) for path in directories} == { + read_reference(environment.environment_dir).sha256, + } + record = second.resource_record.read_bytes() + await environment.stop(delete=True) + assert second.resource_record.read_bytes() == record + assert set(controller.ranges) == {second.range_id} + assert len(controller.installed_roles) == 1 + result = await second.exec('echo proof') + assert result.return_code == 7 + await ludus_resources.recover(second.resource_record) + assert not controller.ranges + assert not controller.installed_roles + await second.stop(delete=True) + assert not controller.ranges + assert not controller.installed_roles + + asyncio.run(exercise()) + + +def test_failed_cleanup_preserves_the_provisioning_error( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + controller.state = 'ERROR' + controller.fail_delete = True + with pytest.raises(RuntimeError, match=': ERROR') as error: + asyncio.run(environment.start(force_build=False)) + assert any('cleanup also failed (RuntimeError)' in note for note in error.value.__notes__) + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == 'owned' + assert not record.roles + controller.fail_delete = False + asyncio.run(ludus_resources.recover(environment.resource_record)) + assert not controller.ranges + + +@pytest.mark.parametrize('delete_fails', [False, True]) +def test_cleanup_reconciles_abort_failure_with_the_delete_outcome( + environment: runner.LudusEnvironment, + controller: Controller, + delete_fails: bool, +) -> None: + asyncio.run(environment.start(force_build=False)) + controller.state = 'DEPLOYING' + controller.fail_abort = True + controller.fail_delete = delete_fails + if delete_fails: + with pytest.raises(RuntimeError, match='cleanup incomplete') as error: + asyncio.run(environment.stop(delete=True)) + assert 'abort ' in str(error.value) + assert 'range ' in str(error.value) + else: + asyncio.run(environment.stop(delete=True)) + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == ('owned' if delete_fails else 'absent') + assert not record.roles + controller.fail_abort = controller.fail_delete = False + asyncio.run(ludus_resources.recover(environment.resource_record)) + assert not controller.ranges + + +def test_cleanup_reconciles_a_previously_owned_range_that_is_already_gone( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + asyncio.run(environment.start(force_build=False)) + controller.ranges.clear() + asyncio.run(environment.stop(delete=True)) + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == 'absent' + assert not record.roles + assert 'DELETE range' not in controller.events + + +def test_cleanup_keeps_role_identity_if_removal_and_reconciliation_both_fail( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + asyncio.run(environment.start(force_build=False)) + controller.fail_role_remove = controller.fail_role_lookup = True + with pytest.raises(RuntimeError, match='cleanup incomplete'): + asyncio.run(environment.stop(delete=True)) + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == 'absent' + assert len(record.roles) == 1 + controller.fail_role_remove = controller.fail_role_lookup = False + asyncio.run(ludus_resources.recover(environment.resource_record)) + recovered = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert not recovered.roles diff --git a/tests/ludus/test_client.py b/tests/ludus/test_client.py new file mode 100644 index 0000000..5f079eb --- /dev/null +++ b/tests/ludus/test_client.py @@ -0,0 +1,178 @@ +"""Validate API boundaries without infrastructure or controller credentials.""" + +import asyncio + +import httpx +import pytest + +from scopebench_ludus.client import LudusAPIError, LudusClient + + +@pytest.mark.parametrize( + 'response', + [ + httpx.Response(403, json={'error': 'secret-password'}), + httpx.Response(200, json={'errors': ['secret-password']}), + httpx.Response(200, json={'success': False, 'result': 'secret-password'}), + httpx.Response(200, text='invalid JSON secret-password'), + ], +) +def test_api_failures_never_expose_response_bodies(response: httpx.Response) -> None: + def request(_request: httpx.Request) -> httpx.Response: + return response + + async def exercise() -> None: + client = LudusClient( + 'https://example.test', 'secret-key', transport=httpx.MockTransport(request) + ) + try: + with pytest.raises(LudusAPIError, match='Ludus') as failure: + _ = await client.request('GET', '') + assert 'secret-password' not in str(failure.value) + assert failure.value.__cause__ is None + finally: + await client.close() + + asyncio.run(exercise()) + + +def test_range_requests_require_an_explicit_valid_id() -> None: + def unexpected(_request: httpx.Request) -> httpx.Response: + raise AssertionError('An unscoped range request reached the server') + + async def exercise() -> None: + client = LudusClient( + 'https://example.test', 'secret-key', transport=httpx.MockTransport(unexpected) + ) + try: + for identity in ('', '../range', 'default?rangeID=other'): + with pytest.raises(ValueError, match='explicit Ludus range ID'): + _ = await client.inventory(identity) + with pytest.raises(ValueError, match='explicit Ludus range ID'): + _ = await client.request('DELETE', 'range') + finally: + await client.close() + + asyncio.run(exercise()) + + +@pytest.mark.parametrize('path', ['https://other.test/user', '//other.test/user', '../user']) +def test_raw_requests_cannot_send_the_key_to_another_server(path: str) -> None: + def unexpected(_request: httpx.Request) -> httpx.Response: + raise AssertionError('An out-of-scope request reached HTTP') + + async def exercise() -> None: + async with LudusClient( + 'https://example.test', 'secret-key', transport=httpx.MockTransport(unexpected) + ) as client: + with pytest.raises(ValueError, match='relative'): + _ = await client.request('GET', path) + + asyncio.run(exercise()) + + +@pytest.mark.parametrize( + 'payload', + [ + {'rangeID': 'OTHER', 'rangeState': 'SUCCESS', 'name': 'n', 'description': 'd'}, + {'rangeID': 'TEST', 'rangeState': 42, 'name': 'n', 'description': 'd'}, + {'rangeID': 'TEST', 'rangeState': 'SUCCESS'}, + ], +) +def test_status_rejects_unexpected_or_invalid_range_identity(payload: dict[str, object]) -> None: + def request(request: httpx.Request) -> httpx.Response: + assert request.url.params['rangeID'] == 'TEST' + return httpx.Response(200, json=payload) + + async def exercise() -> None: + async with LudusClient( + 'https://example.test', 'secret-key', transport=httpx.MockTransport(request) + ) as client: + with pytest.raises(LudusAPIError, match='range'): + _ = await client.status('TEST') + + asyncio.run(exercise()) + + +def test_range_endpoints_match_the_ludus_v2_wire_contract() -> None: + import json + + calls: list[httpx.Request] = [] + + def request(request: httpx.Request) -> httpx.Response: + calls.append(request) + assert request.headers['X-API-KEY'] == 'secret-key' + path = request.url.path + if path.endswith('/user'): + return httpx.Response(200, json=[{'userID': 'JD'}]) + if path.endswith('/ansibleinventory'): + return httpx.Response(200, json={'result': 'all:\n hosts: {}'}) + if path.endswith('/range') and request.method == 'GET': + return httpx.Response( + 200, + json={ + 'rangeID': 'TEST', + 'rangeState': 'SUCCESS', + 'name': 'test', + 'description': 'test', + }, + ) + return httpx.Response(201, json={'result': 'ok'}) + + async def exercise() -> None: + async with LudusClient( + 'https://example.test', 'secret-key', transport=httpx.MockTransport(request) + ) as client: + assert await client.current_user() == 'JD' + await client.create_range('TEST', 'test', 'test', 'JD') + await client.configure_range('TEST', b'ludus: []\n') + await client.deploy('TEST') + assert (await client.status('TEST')).state == 'SUCCESS' + assert await client.inventory('TEST') == 'all:\n hosts: {}' + await client.abort('TEST') + await client.power_off('TEST') + await client.delete_range('TEST') + + asyncio.run(exercise()) + assert [(r.method, r.url.path) for r in calls] == [ + ('GET', '/api/v2/user'), + ('POST', '/api/v2/ranges/create'), + ('PUT', '/api/v2/range/config'), + ('POST', '/api/v2/range/deploy'), + ('GET', '/api/v2/range'), + ('GET', '/api/v2/range/ansibleinventory'), + ('POST', '/api/v2/range/abort'), + ('PUT', '/api/v2/range/poweroff'), + ('DELETE', '/api/v2/range'), + ] + assert json.loads(calls[1].content) == { + 'rangeID': 'TEST', + 'name': 'test', + 'description': 'test', + 'userID': ['JD'], + } + assert all(r.url.params['rangeID'] == 'TEST' for r in calls[2:]) + assert calls[-1].url.params['force'] == 'true' + assert b'ludus: []\n' in calls[2].content + assert json.loads(calls[-2].content) == {'machines': ['all']} + + +def test_transport_errors_hide_secrets_and_do_not_retry_mutations() -> None: + calls = 0 + + def request(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + raise httpx.ReadTimeout('secret-key secret-password', request=request) + + async def exercise() -> None: + async with LudusClient( + 'https://example.test', 'secret-key', transport=httpx.MockTransport(request) + ) as client: + with pytest.raises(LudusAPIError) as failure: + await client.deploy('TEST') + assert 'secret' not in str(failure.value) + assert failure.value.__cause__ is None + + asyncio.run(exercise()) + assert calls == 1 diff --git a/tests/ludus/test_environment.py b/tests/ludus/test_environment.py new file mode 100644 index 0000000..383c33b --- /dev/null +++ b/tests/ludus/test_environment.py @@ -0,0 +1,292 @@ +"""Exercise generic range, HTTP ownership, and Ansible transport contracts.""" + +from __future__ import annotations + +import asyncio +import shutil +from typing import cast + +import pytest +import yaml +from harbor.models.task.config import ( # pyright: ignore[reportMissingTypeStubs] + TaskOS, +) + +from scopebench_contribution.runtime import ( + runtime_reference, +) +from scopebench_ludus import environment as runner +from scopebench_ludus.resources import LudusResources +from tests.ludus.controller import Controller + + +def test_lifecycle_scopes_http_mutations_and_keeps_credentials_private( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + with pytest.raises(ValueError, match='not been loaded'): + _ = environment.config + asyncio.run(environment.stop(delete=True)) + assert not controller.calls + asyncio.run(environment.start(force_build=True)) + assert environment.config.interaction.mode == 'shell' + assert not environment.config.interaction.preseeded_callback + count = len(controller.calls) + with pytest.raises(ValueError, match='already started'): + asyncio.run(environment.start(force_build=False)) + assert len(controller.calls) == count + setup = controller.plays[0][0] + assert set(cast(list[str], setup['loop'])) >= { + 'C:/logs/agent', + 'C:/logs/verifier', + 'C:/logs/artifacts', + } + private = controller.inventory.parent + assert controller.inventory.stat().st_mode & 0o777 == 0o600 + assert all('LUDUS_API_KEY' not in env for env in controller.environments) + assert 'controller-only-secret' not in environment.resource_record.read_text() + asyncio.run(environment.stop(delete=True)) + for request in controller.calls: + if request.url.path.startswith('/api/v2/range/') or request.url.path == '/api/v2/range': + assert request.url.params['rangeID'] == environment.range_id + assert not private.exists() + assert not controller.ranges + assert controller.removed_roles == [f'scopebench_{environment.range_id}_fixture_role'.lower()] + count = len(controller.calls) + asyncio.run(environment.stop(delete=True)) + assert len(controller.calls) == count + + +@pytest.mark.parametrize('failure', ['deployment', 'playbook', 'empty', 'ambiguous']) +def test_failed_start_cleans_partial_resources( + environment: runner.LudusEnvironment, + controller: Controller, + failure: str, +) -> None: + if failure == 'deployment': + controller.state = 'ERROR' + elif failure == 'playbook': + controller.fail_play = True + else: + controller.selected_hosts = [] if failure == 'empty' else ['executor', 'other'] + with pytest.raises((RuntimeError, ValueError)): + asyncio.run(environment.start(force_build=False)) + assert not controller.ranges + assert controller.removed_roles + record = LudusResources.model_validate_json(environment.resource_record.read_bytes()) + assert record.state == 'absent' + assert not record.roles + + +def test_provider_rejects_backend_mismatch( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + reference = environment.environment_dir / 'reference.json' + _ = reference.write_text(reference.read_text().replace('"ludus"', '"cloud"')) + with pytest.raises(ValueError, match='different environment backend'): + _ = runner.LudusEnvironment( + environment_dir=environment.environment_dir, + environment_name='mismatched', + session_id='mismatched-session', + trial_paths=environment.trial_paths, + task_env_config=environment.task_env_config, + ) + assert not controller.calls + + +@pytest.mark.parametrize('windows', [False, True]) +def test_arbitrary_native_topology_preserves_all_vms_and_variables( + environment: runner.LudusEnvironment, + controller: Controller, + windows: bool, +) -> None: + runtime = ( + environment.environment_dir.parents[4] / 'environments/netpen/windows/provider-fixture' + ) + config_path = runtime / 'environment/ludus/range-config.yml' + config = cast(dict[str, object], yaml.safe_load(config_path.read_text())) + machines = cast(list[dict[str, object]], config['ludus']) + machines.extend( + [ + { + 'vm_name': '{{ range_id }}-dc', + 'windows': {'domain': 'example.test'}, + 'roles': ['external.role'], + }, + { + 'vm_name': '{{ range_id }}-linux', + 'linux': True, + 'roles': [], + 'role_vars': {'literal': '{{ untouched }}'}, + }, + ] + ) + config['network'] = {'rules': [{'vlan_src': 10, 'vlan_dst': 20, 'ports': 'all'}]} + _ = config_path.write_text(yaml.safe_dump(config)) + provider = runtime / 'environment/provider.toml' + _ = provider.write_text( + provider.read_text() + .replace('mode = "shell"', 'mode = "mythic-apollo"') + .replace('preseeded_callback = false', 'preseeded_callback = true') + ) + _ = (environment.environment_dir / 'reference.json').write_text( + runtime_reference(runtime.parents[3], runtime).model_dump_json() + ) + environment.task_env_config.os = TaskOS.WINDOWS if windows else TaskOS.LINUX + controller.windows = windows + asyncio.run(environment.start(force_build=False)) + deployed = controller.range_configs[0] + assert cast(list[dict[str, object]], deployed['ludus'])[1:] == machines[1:] + assert deployed['network'] == config['network'] + result = asyncio.run( + environment.exec( + 'echo {{ literal }}', cwd='C:/proof' if windows else '/tmp/proof', user='agent' + ) + ) + assert result.return_code == 7 + task = controller.plays[-1][0] + assert ('ansible.windows.win_shell' if windows else 'ansible.builtin.shell') in task + assert cast(dict[str, object], task['vars'])['ansible_become_method'] == ( + 'runas' if windows else 'sudo' + ) + assert 'hosts: executor' in controller.raw_playbooks[-1] + asyncio.run(environment.stop(delete=True)) + + +@pytest.mark.parametrize('bundled', [False, True]) +def test_installed_native_roles_and_variables_are_preserved_alongside_bundled_roles( + environment: runner.LudusEnvironment, controller: Controller, bundled: bool +) -> None: + runtime = ( + environment.environment_dir.parents[4] / 'environments/netpen/windows/provider-fixture' + ) + provider = runtime / 'environment/provider.toml' + if not bundled: + _ = provider.write_text( + provider.read_text().replace( + 'local_roles = ["ansible/roles/fixture_role"]', 'local_roles = []' + ) + ) + path = runtime / 'environment/ludus/range-config.yml' + config = cast(dict[str, object], yaml.safe_load(path.read_text())) + host = cast(list[dict[str, object]], config['ludus'])[0] + dependency = 'fixture_role' if bundled else 'badsectorlabs.ludus_adcs' + roles: list[object] = [ + 'badsectorlabs.ludus_adcs', + { + 'name': 'existing_role', + 'depends_on': [{'vm_name': '{{ range_id }}-windows', 'role': dependency}], + }, + ] + if bundled: + roles.insert(0, {'name': 'fixture_role'}) + host['roles'] = roles + host['role_vars'] = { + 'ludus_adcs_ca_host': '{{ ansible_hostname }}', + 'nested': {'items': [1, False]}, + 'literal_role_name': 'fixture_role', + } + config['global_role_vars'] = {'global_marker': '{{ range_id }}'} + _ = path.write_text(yaml.safe_dump(config)) + _ = (environment.environment_dir / 'reference.json').write_text( + runtime_reference(runtime.parents[3], runtime).model_dump_json() + ) + asyncio.run(environment.start(force_build=False)) + native = f'scopebench_{environment.range_id}_fixture_role'.lower() + if bundled: + roles[0] = {'name': native} + cast(dict[str, object], roles[2])['depends_on'] = [ + {'vm_name': '{{ range_id }}-windows', 'role': native} + ] + assert 'Inert role fixture' in controller.role_tasks[0] + assert ( + controller.events.index('PUT ansible/role/fromtar') + < controller.events.index('POST range/deploy') + < controller.events.index('transport') + ) + assert controller.range_configs == [config] + assert bool(controller.role_tasks) == bundled + asyncio.run(environment.stop(delete=True)) + assert controller.removed_roles == ([native] if bundled else []) + + +def test_failed_upload_cleans_the_pending_role_without_deploying( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + controller.fail_role = True + with pytest.raises(RuntimeError, match='HTTP 500') as failure: + asyncio.run(environment.start(force_build=False)) + assert 'fixture secret' not in str(failure.value) + assert not controller.ranges + assert controller.removed_roles + assert 'POST range/deploy' not in controller.events + + +@pytest.mark.parametrize( + ('name', 'value'), + [ + ('LUDUS_VERIFY', 'maybe'), + ('LUDUS_URL', 'http://example'), + ('LUDUS_DEPLOY_TIMEOUT_SEC', 'nan'), + ], +) +def test_invalid_settings_fail_before_api_mutation( + environment: runner.LudusEnvironment, + controller: Controller, + monkeypatch: pytest.MonkeyPatch, + name: str, + value: str, +) -> None: + monkeypatch.setenv(name, value) + with pytest.raises(ValueError, match='Ludus|LUDUS|finite'): + asyncio.run(environment.start(force_build=False)) + assert not controller.calls + + +def test_deployment_deadline_aborts_and_cleans_partial_range( + environment: runner.LudusEnvironment, + controller: Controller, + monkeypatch: pytest.MonkeyPatch, +) -> None: + controller.state = 'DEPLOYING' + monkeypatch.setenv('LUDUS_DEPLOY_TIMEOUT_SEC', '0.001') + with pytest.raises(TimeoutError): + asyncio.run(environment.start(force_build=False)) + assert 'POST range/abort' in controller.events + assert not controller.ranges + assert controller.removed_roles + + +def test_bundled_metadata_dependencies_follow_trial_role_names( + environment: runner.LudusEnvironment, + controller: Controller, +) -> None: + runtime = ( + environment.environment_dir.parents[4] / 'environments/netpen/windows/provider-fixture' + ) + definition = runtime / 'environment' + other = definition / 'ansible/roles/dependent_role' + _ = shutil.copytree(definition / 'ansible/roles/fixture_role', other) + meta = other / 'meta/main.yml' + _ = meta.write_text( + meta.read_text().replace('dependencies: []', 'dependencies: [fixture_role, external.role]') + ) + provider = definition / 'provider.toml' + _ = provider.write_text( + provider.read_text().replace( + '["ansible/roles/fixture_role"]', + '["ansible/roles/fixture_role", "ansible/roles/dependent_role"]', + ) + ) + _ = (environment.environment_dir / 'reference.json').write_text( + runtime_reference(runtime.parents[3], runtime).model_dump_json() + ) + asyncio.run(environment.start(force_build=False)) + uploaded = cast(dict[str, object], yaml.safe_load(controller.role_metadata[1])) + assert uploaded['dependencies'] == [ + f'scopebench_{environment.range_id}_fixture_role'.lower(), + 'external.role', + ] + asyncio.run(environment.stop(delete=True)) diff --git a/tests/ludus/test_guest.py b/tests/ludus/test_guest.py new file mode 100644 index 0000000..bf0d64a --- /dev/null +++ b/tests/ludus/test_guest.py @@ -0,0 +1,67 @@ +"""Exercise generic range, HTTP ownership, and Ansible transport contracts.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import cast + +import pytest +from harbor.environments.base import ExecResult # pyright: ignore[reportMissingTypeStubs] +from harbor.models.task.config import ( # pyright: ignore[reportMissingTypeStubs] + TaskOS, +) + +from scopebench_ludus import environment as runner +from tests.ludus.controller import Controller + + +def test_proxy_is_only_passed_to_ansible_and_scoped_exec_is_literal( + environment: runner.LudusEnvironment, + controller: Controller, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv('LUDUS_WINRM_PROXY', 'http://127.0.0.1:22148') + asyncio.run(environment.start(force_build=False)) + environment.default_user = 'agent' + with environment.scoped_exec_env({'SB_PROOF': '{{ scoped }}'}): + result = asyncio.run( + environment.exec('echo {{ literal }}', env={'SB_PROOF': 'per-command'}, cwd=r'C:\proof') + ) + task = controller.plays[-1][0] + assert task['environment'] == {'SB_PROOF': '{{ scoped }}'} + assert cast(dict[str, object], task['vars'])['ansible_become_user'] == 'agent' + assert cast(dict[str, str], task['args'])['chdir'] == r'C:\proof' + assert '!unsafe' in controller.raw_playbooks[-1] + assert result == ExecResult(return_code=7, stdout='fixture stdout\r\n', stderr='fixture stderr') + for process in controller.processes: + if process[0] == 'ansible-playbook': + assert json.loads(process[process.index('--extra-vars') + 1]) == { + 'ansible_winrm_proxy': 'http://127.0.0.1:22148' + } + with pytest.raises(ValueError, match='numeric UIDs'): + _ = asyncio.run(environment.exec('whoami', user=1000)) + asyncio.run(environment.stop(delete=True)) + + +@pytest.mark.parametrize('windows', [False, True]) +def test_directory_download_preserves_empty_dirs_and_rejects_traversal( + environment: runner.LudusEnvironment, + controller: Controller, + tmp_path: Path, + windows: bool, +) -> None: + controller.windows = windows + environment.task_env_config.os = TaskOS.WINDOWS if windows else TaskOS.LINUX + asyncio.run(environment.start(force_build=False)) + root = r'C:\proof' if windows else '/proof' + separator = '\\' if windows else '/' + controller.entries = [{'path': root + separator + 'empty', 'isdir': True}] + asyncio.run(environment.download_dir(root, tmp_path / 'download')) + assert (tmp_path / 'download/empty').is_dir() + controller.entries = [{'path': root + separator + '..' + separator + 'escape', 'isdir': False}] + with pytest.raises(ValueError, match='Invalid path'): + asyncio.run(environment.download_dir(root, tmp_path / 'download')) + assert not (tmp_path / 'escape').exists() + asyncio.run(environment.stop(delete=True)) diff --git a/tests/ludus/test_inventory.py b/tests/ludus/test_inventory.py new file mode 100644 index 0000000..3db6d9c --- /dev/null +++ b/tests/ludus/test_inventory.py @@ -0,0 +1,74 @@ +"""Validate inventory shape once, then select hosts using typed connection data.""" + +import json + +import pytest +from pydantic import JsonValue, ValidationError + +from scopebench_ludus.ansible.inventory import Inventory + + +def inventory_document() -> dict[str, JsonValue]: + return { + '_meta': { + 'hostvars': { + 'linux': {'ansible_connection': 'ssh', 'ansible_password': 'fixture-password'}, + 'windows': {'ansible_connection': 'winrm'}, + } + }, + 'executor': {'children': ['nested']}, + 'nested': {'hosts': ['linux'], 'children': ['executor']}, + 'all': {'hosts': ['linux', 'windows']}, + } + + +def test_inventory_resolves_nested_cycles_and_keeps_only_connection_kind() -> None: + inventory = Inventory.from_json(json.dumps(inventory_document())) + assert inventory.execution_host('executor', expected_windows=False) == 'linux' + assert inventory.execution_host('windows', expected_windows=True) == 'windows' + assert inventory.groups['executor'].children == ['nested'] + assert 'fixture-password' not in inventory.model_dump_json() + + +@pytest.mark.parametrize('selector', ['all', 'missing', 'linux:windows', '*']) +def test_inventory_requires_one_literal_execution_host(selector: str) -> None: + inventory = Inventory.from_json(json.dumps(inventory_document())) + with pytest.raises(ValueError, match='exactly one'): + _ = inventory.execution_host(selector, expected_windows=False) + + +def test_inventory_rejects_execution_os_mismatch() -> None: + inventory = Inventory.from_json(json.dumps(inventory_document())) + with pytest.raises(ValueError, match='differs'): + _ = inventory.execution_host('windows', expected_windows=False) + + +@pytest.mark.parametrize( + 'invalid', + [ + {'_meta': {'hostvars': []}}, + {'_meta': {'hostvars': {'linux': {'ansible_connection': 7}}}}, + {'_meta': {'hostvars': {}}, 'group': {'hosts': ['linux', 7]}}, + {'_meta': {'hostvars': {}}, 'group': {'children': 'not-a-list'}}, + {'group': {'hosts': ['linux']}}, + ], +) +def test_inventory_rejects_malformed_connection_and_group_data( + invalid: dict[str, JsonValue], +) -> None: + with pytest.raises(ValidationError): + _ = Inventory.from_json(json.dumps(invalid)) + + +def test_inventory_parse_errors_hide_credential_bearing_input() -> None: + with pytest.raises(ValidationError) as error: + _ = Inventory.from_json('{"_meta": "fixture-password" INVALID}') + assert 'fixture-password' not in str(error.value) + + +def test_inventory_rejects_group_host_without_connection_metadata() -> None: + inventory = Inventory.from_json( + json.dumps({'_meta': {'hostvars': {}}, 'executor': {'hosts': ['missing']}}) + ) + with pytest.raises(ValueError, match='no connection variables'): + _ = inventory.execution_host('executor', expected_windows=False) diff --git a/tests/ludus/test_process.py b/tests/ludus/test_process.py new file mode 100644 index 0000000..d6280a3 --- /dev/null +++ b/tests/ludus/test_process.py @@ -0,0 +1,128 @@ +"""Exercise generic range, HTTP ownership, and Ansible transport contracts.""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path + +import psutil +import pytest + +from scopebench_ludus.ansible import process as transport +from scopebench_ludus.ansible.process import ProcessResult + + +def test_real_controller_process_returns_outputs_and_exit_status() -> None: + result = asyncio.run( + transport.run_process( + [ + sys.executable, + '-c', + 'import sys; print("stdout"); print("stderr", file=sys.stderr); sys.exit(7)', + ], + timeout=10, + ) + ) + assert result == ProcessResult(stdout='stdout\n', stderr='stderr\n', return_code=7) + + +@pytest.mark.parametrize('cancel', [False, True]) +def test_controller_abort_kills_detached_worker_tree(tmp_path: Path, cancel: bool) -> None: + # Ansible workers detach with setsid; a process-group kill alone misses them. + worker_pid = tmp_path / 'worker.pid' + grandchild_pid = tmp_path / 'grandchild.pid' + worker = ( + 'import subprocess, sys, time; from pathlib import Path; ' + 'child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], ' + 'start_new_session=True); ' + f'Path({str(grandchild_pid)!r}).write_text(str(child.pid)); time.sleep(30)' + ) + controller = ( + 'import subprocess, sys, time; from pathlib import Path; ' + f'child = subprocess.Popen([sys.executable, "-c", {worker!r}], ' + 'start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); ' + f'Path({str(worker_pid)!r}).write_text(str(child.pid)); time.sleep(30)' + ) + + async def exercise() -> None: + operation = asyncio.create_task( + transport.run_process([sys.executable, '-c', controller], timeout=5) + ) + try: + async with asyncio.timeout(4): + while not grandchild_pid.exists(): + await asyncio.sleep(0.01) + if cancel: + _ = operation.cancel() + with pytest.raises(asyncio.CancelledError if cancel else TimeoutError): + _ = await operation + finally: + if not operation.done(): + _ = operation.cancel() + with pytest.raises(asyncio.CancelledError): + _ = await operation + + try: + asyncio.run(exercise()) + for path in (grandchild_pid, worker_pid): + try: + _ = psutil.Process(int(path.read_text())).wait(timeout=3) + except psutil.NoSuchProcess: + pass + finally: + # Keep a failed regression from leaving its synthetic workers behind. + for path in (grandchild_pid, worker_pid): + if path.exists(): + try: + psutil.Process(int(path.read_text())).kill() + except psutil.NoSuchProcess: + pass + + +@pytest.mark.parametrize(('cancel', 'detached'), [(False, False), (True, False), (False, True)]) +def test_controller_abort_closes_inherited_pipes_after_parent_exit( + tmp_path: Path, cancel: bool, detached: bool +) -> None: + parent_pid = tmp_path / 'parent.pid' + worker_pid = tmp_path / 'worker.pid' + # An orphan can retain the pipes even after it escapes the parent's group. + controller = ( + 'import os, subprocess, sys; from pathlib import Path; ' + f'Path({str(parent_pid)!r}).write_text(str(os.getpid())); ' + 'child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], ' + f'start_new_session={detached}); ' + f'Path({str(worker_pid)!r}).write_text(str(child.pid))' + ) + + async def exercise() -> None: + operation = asyncio.create_task( + transport.run_process([sys.executable, '-c', controller], timeout=5) + ) + try: + async with asyncio.timeout(4): + while not worker_pid.exists() or psutil.pid_exists(int(parent_pid.read_text())): + await asyncio.sleep(0.01) + if cancel: + _ = operation.cancel() + done, _ = await asyncio.wait({operation}, timeout=7) + assert operation in done, 'Aborting the controller waited for an inherited output pipe' + with pytest.raises(asyncio.CancelledError if cancel else TimeoutError): + _ = await operation + if not detached: + try: + _ = psutil.Process(int(worker_pid.read_text())).wait(timeout=3) + except psutil.NoSuchProcess: + pass + finally: + if worker_pid.exists(): + try: + psutil.Process(int(worker_pid.read_text())).kill() + except psutil.NoSuchProcess: + pass + if not operation.done(): + _ = operation.cancel() + with pytest.raises(asyncio.CancelledError): + _ = await operation + + asyncio.run(exercise()) diff --git a/tests/ludus_helpers.py b/tests/ludus_helpers.py new file mode 100644 index 0000000..9594bca --- /dev/null +++ b/tests/ludus_helpers.py @@ -0,0 +1,43 @@ +"""Build inert Ludus definitions beside task bindings for provider tests.""" + +from pathlib import Path + +import yaml + +from scopebench_contribution.runtime import runtime_reference +from tests.benchmark_helpers import write + +IMPORT_PATH = 'scopebench_ludus.environment:LudusEnvironment' + + +def prepare_environment(root: Path) -> Path: + """Generate an inert definition for mocked providers, without runnable tasks.""" + scenario = root / 'environments/netpen/windows/provider-fixture' + runtime = scenario / 'environment' + write(scenario / 'scenario.toml', 'backend = "ludus"\nversion = "0.1.0"\n') + write( + runtime / 'provider.toml', + 'local_roles = ["ansible/roles/fixture_role"]\nhost = "scopebench"\n' + + '[interaction]\nmode = "shell"\npreseeded_callback = false\n', + ) + write( + runtime / 'ludus/range-config.yml', + yaml.safe_dump( + { + 'ludus': [ + { + 'vm_name': '{{ range_id }}-windows', + 'template': 'unavailable-test-template', + 'roles': ['fixture_role'], + 'role_vars': {'fixture_marker': 'inert'}, + } + ] + } + ), + ) + role = runtime / 'ansible/roles/fixture_role' + write(role / 'meta/main.yml', 'dependencies: []\n') + write(role / 'tasks/main.yml', '- name: Inert role fixture\n ansible.builtin.debug:\n') + target = root / 'tasks/netpen/windows/provider-fixture/environment' + write(target / 'reference.json', runtime_reference(root, scenario).model_dump_json()) + return target diff --git a/tests/test_environments.py b/tests/test_environments.py index e3831f1..77d0bbc 100644 --- a/tests/test_environments.py +++ b/tests/test_environments.py @@ -1,28 +1,19 @@ -"""Portable definitions and provider lifecycle contracts use inert VM fixtures.""" +"""Portable definitions, provider configuration, and read-only inspection contracts.""" -import asyncio -import io import os import shutil import subprocess -import tarfile from pathlib import Path -from typing import override +import httpx import pytest -from harbor.environments.base import ExecResult # pyright: ignore[reportMissingTypeStubs] -from harbor.models.task.config import EnvironmentConfig # pyright: ignore[reportMissingTypeStubs] -from harbor.models.trial.paths import TrialPaths # pyright: ignore[reportMissingTypeStubs] -from scopebench_environments import definitions, ludus -from scopebench_environments.__main__ import package, run -from scopebench_environments.base import ( - BoundEnvironmentBase, - CloudEnvironmentBase, - LudusEnvironmentBase, -) -from scopebench_contribution import controls, pairs +from scopebench_contribution import controls, pairs, runtime as definitions from scopebench_contribution.native import TaskConfig, read_task +from scopebench_contribution.runtime import package, run +from scopebench_contribution.scenarios import Backend, scenario_path +from scopebench_ludus import __main__ as cli, config as ludus +from scopebench_ludus.client import LudusAPIError, LudusClient from tests.benchmark_helpers import SCENARIO, task_path, write @@ -36,7 +27,7 @@ def vm_root(image_root: Path) -> Path: write(scenario / 'environment/provider.toml', '[interaction]\nmode = "shell"\n') write(scenario / 'environment/assets/readme.txt', 'fixture\n') (scenario / 'environment/assets/readme.txt').chmod(0o755) - binding = definitions.binding_for(image_root, scenario) + binding = definitions.runtime_reference(image_root, scenario) for manifest in (image_root / 'lite/tasks/web').glob('*/task.toml'): write( manifest, @@ -50,24 +41,27 @@ def vm_root(image_root: Path) -> Path: return image_root -def test_reproducible_bundle_preserves_runtime_and_modes(vm_root: Path, tmp_path: Path) -> None: +def test_raw_packages_preserve_runtime_and_modes(vm_root: Path, tmp_path: Path) -> None: runtime = vm_root / 'lite/environments/web' / SCENARIO / 'environment' - first, second = tmp_path / 'one.tar', tmp_path / 'two.tar' - definitions.write_bundle(runtime, first) + first, second = tmp_path / 'one', tmp_path / 'two' + assert package(vm_root, first, 'ludus') == 2 os.utime(runtime / 'assets/readme.txt', (100, 100)) - definitions.write_bundle(runtime, second) - assert first.read_bytes() == second.read_bytes() - destination = tmp_path / 'unpacked' - definitions.unpack_bundle(first, destination) - assert definitions.definition_digest(destination) == definitions.definition_digest(runtime) + assert package(vm_root, second, 'ludus') == 2 + first_runtime = first / 'lite/environments/web' / SCENARIO / 'environment' + second_runtime = second / 'lite/environments/web' / SCENARIO / 'environment' + assert definitions.runtime_digest(first_runtime) == definitions.runtime_digest(second_runtime) + destination = first_runtime + assert definitions.runtime_digest(destination) == definitions.runtime_digest(runtime) assert (destination / 'assets/readme.txt').stat().st_mode & 0o111 assert not list(destination.rglob('solve*')) + assert not list(first.rglob('*.tar')) + assert not (first_runtime.parent / 'solution').exists() @pytest.mark.parametrize('name', ['', '.', '..', 'a/b', 'a\\b']) def test_scenario_identity_cannot_escape(tmp_path: Path, name: str) -> None: with pytest.raises(ValueError, match='Invalid scenario name'): - _ = definitions.scenario_path(tmp_path, 'web', name) + _ = scenario_path(tmp_path, 'web', name) @pytest.mark.parametrize('defect', ['missing', 'root-link', 'empty', 'file-link', 'fifo']) @@ -84,63 +78,35 @@ def test_runtime_definitions_reject_special_files(tmp_path: Path, defect: str) - elif defect == 'fifo': os.mkfifo(directory / 'pipe') with pytest.raises(ValueError, match='runtime definition|ordinary files|Empty'): - _ = definitions.definition_files(directory) - - -@pytest.mark.parametrize( - 'name', ['/outside', '../outside', 'a/../outside', './a', 'a\\b', '', 'a//b'] -) -def test_archives_reject_unsafe_paths(tmp_path: Path, name: str) -> None: - bundle = tmp_path / 'definition.tar' - with tarfile.open(bundle, 'w') as archive: - info = tarfile.TarInfo(name) - info.size = 1 - archive.addfile(info, io.BytesIO(b'x')) - with pytest.raises(ValueError, match='Invalid runtime definition archive'): - definitions.unpack_bundle(bundle, tmp_path / 'output') - - -@pytest.mark.parametrize('kind', ['link', 'directory', 'duplicate']) -def test_archives_reject_nonfiles_and_duplicate_entries(tmp_path: Path, kind: str) -> None: - bundle = tmp_path / 'definition.tar' - with tarfile.open(bundle, 'w') as archive: - info = tarfile.TarInfo('asset') - info.type = ( - tarfile.SYMTYPE - if kind == 'link' - else tarfile.DIRTYPE - if kind == 'directory' - else tarfile.REGTYPE - ) - archive.addfile(info) - if kind == 'duplicate': - archive.addfile(info) - with pytest.raises(ValueError, match='Invalid runtime definition archive'): - definitions.unpack_bundle(bundle, tmp_path / 'output') + _ = definitions.runtime_files(directory) def test_resolution_from_checkout_and_packaged_tasks(vm_root: Path, tmp_path: Path) -> None: native = task_path(vm_root) / 'environment' - binding, runtime = definitions.resolve_definition(native, tmp_path / 'checked-out') - assert binding.sha256 == definitions.definition_digest(runtime) + binding, runtime = definitions.snapshot_runtime(native, tmp_path / 'checked-out') + assert binding.sha256 == definitions.runtime_digest(runtime) output = tmp_path / 'portable' assert package(vm_root, output, 'ludus') == 2 + assert len(list((output / 'lite/environments').rglob('scenario.toml'))) == 1 + assert sorted(path.name for path in task_path(output).joinpath('environment').iterdir()) == [ + 'reference.json' + ] # Portable tasks remain resolvable after the source checkout disappears. shutil.rmtree(vm_root / 'lite/environments/web') portable = task_path(output) / 'environment' - copied_binding, copied = definitions.resolve_definition(portable, tmp_path / 'distributed') + copied_binding, copied = definitions.snapshot_runtime(portable, tmp_path / 'distributed') assert copied_binding == binding - assert definitions.definition_digest(copied) == binding.sha256 + assert definitions.runtime_digest(copied) == binding.sha256 def test_lite_resolution_prefers_its_own_collection(vm_root: Path, tmp_path: Path) -> None: main = vm_root / 'environments/web' / SCENARIO _ = shutil.copytree(vm_root / 'lite/environments/web' / SCENARIO, main) write(main / 'environment/assets/readme.txt', 'different main definition\n') - binding, runtime = definitions.resolve_definition( + binding, runtime = definitions.snapshot_runtime( task_path(vm_root) / 'environment', tmp_path / 'resolved' ) - assert definitions.definition_digest(runtime) == binding.sha256 + assert definitions.runtime_digest(runtime) == binding.sha256 assert (runtime / 'assets/readme.txt').read_text() == 'fixture\n' @@ -156,8 +122,8 @@ def test_packaging_main_tasks_uses_main_scenarios(vm_root: Path, tmp_path: Path) output = tmp_path / 'portable' assert package(vm_root, output, 'ludus') == 2 native = output / 'tasks/web' / f'{SCENARIO}-capability' / 'environment' - binding, runtime = definitions.resolve_definition(native, tmp_path / 'resolved') - assert definitions.definition_digest(runtime) == binding.sha256 + binding, runtime = definitions.snapshot_runtime(native, tmp_path / 'resolved') + assert definitions.runtime_digest(runtime) == binding.sha256 @pytest.mark.parametrize('defect', ['existing', 'dangling-link', 'unavailable', 'fingerprint']) @@ -172,7 +138,7 @@ def test_resolution_fails_closed(vm_root: Path, tmp_path: Path, defect: str) -> else: write(vm_root / 'lite/environments/web' / SCENARIO / 'environment/new.txt', 'changed') with pytest.raises(ValueError, match='already exists|unavailable|fingerprint differs'): - _ = definitions.resolve_definition(task_path(vm_root) / 'environment', destination) + _ = definitions.snapshot_runtime(task_path(vm_root) / 'environment', destination) @pytest.mark.parametrize('defect', ['docker', 'existing', 'none', 'binding']) @@ -180,7 +146,7 @@ def test_packaging_requires_matching_backend_and_binding( vm_root: Path, tmp_path: Path, defect: str ) -> None: output = tmp_path / 'packaged' - backend: definitions.Backend = 'ludus' + backend: Backend = 'ludus' if defect == 'docker': backend = 'docker' elif defect == 'existing': @@ -201,138 +167,14 @@ def test_packaging_requires_matching_backend_and_binding( def test_package_cli_uses_current_checkout(vm_root: Path, tmp_path: Path) -> None: output = tmp_path / 'cli-output' result = subprocess.run( - ['scopebench-environments', 'package', '--backend', 'ludus', '--output', str(output)], + ['scopebench-package', '--backend', 'ludus', '--output', str(output)], cwd=vm_root, text=True, capture_output=True, check=True, ) assert 'Packaged 2 tasks' in result.stdout - assert ( - run(['package', '--backend', 'ludus', '--output', str(tmp_path / 'other')], root=vm_root) - == 0 - ) - - -class FakeLudus(LudusEnvironmentBase): - """A provider that records resource boundaries without connecting to a server.""" - - calls: list[str] - fail: bool = False - fail_cleanup: bool = False - - @override - async def provision( - self, definition: Path, binding: definitions.Binding, *, force_build: bool - ) -> None: - assert definitions.definition_digest(definition) == binding.sha256 - self.calls.append(f'provision:{self.session_id}:{force_build}') - if self.fail: - raise RuntimeError('fixture provisioning failure') - - @override - async def release(self, *, delete: bool) -> None: - self.calls.append(f'release:{delete}') - if self.fail_cleanup: - raise ValueError('fixture cleanup failure') - - @override - async def exec( - self, - command: str, - cwd: str | None = None, - env: dict[str, str] | None = None, - timeout_sec: int | None = None, - user: str | int | None = None, - ) -> ExecResult: - _ = command, cwd, env, timeout_sec, user - return ExecResult(return_code=0) - - @override - async def upload_file(self, source_path: Path | str, target_path: str) -> None: - _ = source_path, target_path - - @override - async def upload_dir(self, source_dir: Path | str, target_dir: str) -> None: - _ = source_dir, target_dir - - @override - async def download_file(self, source_path: str, target_path: Path | str) -> None: - _ = source_path, target_path - - @override - async def download_dir(self, source_dir: str, target_dir: Path | str) -> None: - _ = source_dir, target_dir - - -def provider(vm_root: Path, session: str = 'trial-one') -> FakeLudus: - """Construct the real Harbor base with one task-local reference.""" - instance = FakeLudus( - environment_dir=task_path(vm_root) / 'environment', - environment_name=SCENARIO, - session_id=session, - trial_paths=TrialPaths(vm_root / 'jobs' / session), - task_env_config=EnvironmentConfig(), - ) - instance.calls = [] - return instance - - -def test_ludus_lifecycle_loads_interaction_and_releases_once(vm_root: Path) -> None: - instance = provider(vm_root) - with pytest.raises(ValueError, match='not been configured'): - _ = instance.definition - asyncio.run(instance.stop(delete=True)) - assert not instance.calls - asyncio.run(instance.start(force_build=True)) - assert instance.definition.interaction.mode == 'shell' - assert not instance.definition.interaction.preseeded_callback - with pytest.raises(ValueError, match='already started'): - asyncio.run(instance.start(force_build=False)) - asyncio.run(instance.stop(delete=False)) - asyncio.run(instance.stop(delete=True)) - assert instance.calls == ['provision:trial-one:True', 'release:False'] - assert LudusEnvironmentBase.type() == 'ludus' - assert CloudEnvironmentBase.type() == 'cloud' - BoundEnvironmentBase.configure(instance, vm_root) - - -def test_failed_start_cleans_partial_resources(vm_root: Path) -> None: - instance = provider(vm_root) - instance.fail = True - with pytest.raises(RuntimeError, match='provisioning failure'): - asyncio.run(instance.start(force_build=False)) - asyncio.run(instance.stop(delete=True)) - assert instance.calls == ['provision:trial-one:False', 'release:True'] - - -def test_related_providers_have_separate_definition_storage(vm_root: Path) -> None: - instances = [provider(vm_root, 'agent'), provider(vm_root, 'grader')] - for instance in instances: - instance.trial_paths = TrialPaths(vm_root / 'jobs/shared-trial') - asyncio.run(instance.start(force_build=False)) - directories = list((vm_root / 'jobs/shared-trial/runtime-definitions').iterdir()) - assert len(directories) == 2 - assert {definitions.definition_digest(path) for path in directories} == { - definitions.read_binding(task_path(vm_root) / 'environment').sha256, - } - for instance in instances: - asyncio.run(instance.stop(delete=True)) - - -def test_failed_cleanup_preserves_the_provisioning_error(vm_root: Path) -> None: - instance = provider(vm_root) - instance.fail = instance.fail_cleanup = True - with pytest.raises(RuntimeError, match='provisioning failure') as error: - asyncio.run(instance.start(force_build=False)) - assert any('cleanup also failed (ValueError)' in note for note in error.value.__notes__) - - -def test_provider_rejects_backend_mismatch(vm_root: Path) -> None: - path = task_path(vm_root) / 'environment/reference.json' - write(path, path.read_text().replace('"ludus"', '"cloud"')) - with pytest.raises(ValueError, match='different environment backend'): - _ = provider(vm_root) + assert run(['--backend', 'ludus', '--output', str(tmp_path / 'other')], root=vm_root) == 0 @pytest.mark.parametrize( @@ -342,7 +184,28 @@ def test_ludus_configuration_binds_local_assets(vm_root: Path, setting: str) -> runtime = vm_root / 'lite/environments/web' / SCENARIO / 'environment' write(runtime / 'provider.toml', f'range_config = {setting!r}\n') with pytest.raises(ValueError, match='inside the definition'): - _ = ludus.read_definition(runtime) + _ = ludus.read_config(runtime) + + +@pytest.mark.parametrize('defect', ['escape', 'missing-meta', 'link', 'name', 'duplicate']) +def test_bundled_ludus_roles_are_bounded_and_unambiguous(vm_root: Path, defect: str) -> None: + runtime = vm_root / 'lite/environments/web' / SCENARIO / 'environment' + directory = 'ansible/roles/role' if defect != 'name' else 'ansible/roles/not-a-role' + write(runtime / directory / 'meta/main.yml', 'dependencies: []\n') + directories = [directory] + if defect == 'escape': + directories = ['../outside'] + elif defect == 'missing-meta': + (runtime / directory / 'meta/main.yml').unlink() + elif defect == 'link': + (runtime / directory / 'payload').symlink_to(runtime / 'assets/readme.txt') + elif defect == 'duplicate': + other = 'other/ROLE' + write(runtime / other / 'meta/main.yml', 'dependencies: []\n') + directories.append(other) + write(runtime / 'provider.toml', f'local_roles = {directories!r}\n') + with pytest.raises(ValueError, match='inside the definition|ordinary files|unique identifier'): + _ = ludus.read_config(runtime) def test_preseeded_callback_is_environment_configurable(vm_root: Path) -> None: @@ -351,13 +214,13 @@ def test_preseeded_callback_is_environment_configurable(vm_root: Path) -> None: runtime / 'provider.toml', '[interaction]\nmode = "custom-adapter"\npreseeded_callback = true\nrequired_env = ["CALLBACK_ID"]\n', ) - config = ludus.read_definition(runtime) + config = ludus.read_config(runtime) assert config.interaction.preseeded_callback assert config.interaction.mode == 'custom-adapter' assert config.interaction.required_env == ['CALLBACK_ID'] write(runtime / 'provider.toml', '[interaction]\nrequired_env = ["not a variable"]\n') with pytest.raises(ValueError, match='environment variables'): - _ = ludus.read_definition(runtime) + _ = ludus.read_config(runtime) @pytest.mark.parametrize('backend', ['ludus', 'cloud']) @@ -367,7 +230,7 @@ def test_vm_and_cloud_pairs_share_immutable_bindings(vm_root: Path, backend: str scenario / 'scenario.toml', (scenario / 'scenario.toml').read_text().replace('ludus', backend), ) - binding = definitions.binding_for(vm_root, scenario) + binding = definitions.runtime_reference(vm_root, scenario) for manifest in (vm_root / 'lite/tasks/web').glob('*/task.toml'): write(manifest, manifest.read_text().replace('backend = "ludus"', f'backend = "{backend}"')) write(manifest.parent / 'environment/reference.json', binding.model_dump_json()) @@ -400,53 +263,48 @@ def test_pair_validation_rejects_vm_contract_drift(vm_root: Path, defect: str) - ) def test_ludus_inspection_rejects_invalid_urls(url: str) -> None: with pytest.raises(ValueError, match='HTTPS Ludus URL'): - _ = ludus.inspect_instance(url) + _ = cli.inspect_instance(url) def test_ludus_inspection_requires_external_credential(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv('LUDUS_API_KEY', raising=False) - with pytest.raises(ValueError, match='controller environment'): - _ = ludus.inspect_instance('https://example.test') + with pytest.raises(ValueError, match='API key'): + _ = cli.inspect_instance('https://example.test') @pytest.mark.parametrize('verify', [False, True]) -def test_inspection_uses_only_read_commands_and_redacts_errors( - tmp_path: Path, +def test_inspection_uses_read_only_http_and_hides_response_secrets( monkeypatch: pytest.MonkeyPatch, verify: bool, ) -> None: monkeypatch.setenv('LUDUS_API_KEY', 'fixture-secret') - calls: list[list[str]] = [] + expected_verify = verify + calls: list[httpx.Request] = [] failure = False - def execute( - args: list[str], *, capture_output: bool, text: bool, timeout: int, check: bool - ) -> subprocess.CompletedProcess[str]: - assert capture_output - assert text - assert timeout == 30 - assert not check - assert 'fixture-secret' not in args - assert ('--verify' in args) == verify - calls.append(args) - if not failure and verify and args[-1] == 'version': - return subprocess.CompletedProcess(args, 0, '', 'server 2.0') - return subprocess.CompletedProcess( - args, - int(failure), - '[{}]' if '--json' in args else 'server 2.0', - 'fixture-secret unavailable', + def execute(request: httpx.Request) -> httpx.Response: + calls.append(request) + assert request.method == 'GET' + assert request.headers['X-API-KEY'] == 'fixture-secret' + if failure: + return httpx.Response(401, json={'error': 'fixture-secret unavailable'}) + return httpx.Response( + 200, json=[{}] if request.url.path.endswith('/templates') else {'result': 'server 2.0'} ) - monkeypatch.setattr('scopebench_environments.ludus.subprocess.run', execute) + def client(url: str, api_key: str, *, verify: bool) -> LudusClient: + assert verify is expected_verify + return LudusClient(url, api_key, verify=verify, transport=httpx.MockTransport(execute)) + + monkeypatch.setattr(cli, 'LudusClient', client) assert ( - run( - ['inspect-ludus', '--url', 'https://example.test', *([] if verify else ['--insecure'])], - root=tmp_path, + cli.run( + ['inspect', '--url', 'https://example.test', *([] if verify else ['--insecure'])], ) == 0 ) - assert [call[-1] for call in calls] == ['version', 'list'] + assert [call.url.path for call in calls] == ['/api/v2/', '/api/v2/templates'] failure = True - with pytest.raises(ValueError, match=r'\[redacted\] unavailable'): - _ = ludus.inspect_instance('https://example.test', verify=verify) + with pytest.raises(LudusAPIError, match='HTTP 401') as error: + _ = cli.inspect_instance('https://example.test', verify=verify) + assert 'fixture-secret' not in str(error.value) diff --git a/tests/test_ludus_setup.py b/tests/test_ludus_setup.py new file mode 100644 index 0000000..9442389 --- /dev/null +++ b/tests/test_ludus_setup.py @@ -0,0 +1,32 @@ +"""Controller setup installs modules without contacting Ludus.""" + +import pytest + +from scopebench_ludus.__main__ import run + + +def test_setup_installs_controller_collection_without_server_access( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import subprocess + + calls: list[list[str]] = [] + + def which(_name: str) -> str: + return '/fixture/bin' + + def install( + args: list[str], *, env: dict[str, str], timeout: int, check: bool + ) -> subprocess.CompletedProcess[str]: + assert 'LUDUS_API_KEY' not in env + assert timeout == 300 + assert not check + calls.append(args) + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setenv('LUDUS_API_KEY', 'controller-only-secret') + monkeypatch.setattr('scopebench_ludus.setup.shutil.which', which) + monkeypatch.setattr('scopebench_ludus.setup.subprocess.run', install) + assert run(['setup']) == 0 + assert len(calls) == 1 + assert calls[0][:4] == ['ansible-galaxy', 'collection', 'install', 'ansible.windows:3.2.0'] diff --git a/tests/test_pairs.py b/tests/test_pairs.py index e4d87bc..4bc85eb 100644 --- a/tests/test_pairs.py +++ b/tests/test_pairs.py @@ -8,9 +8,10 @@ from pathlib import Path import pytest -from scopebench_environments.definitions import scenario_paths from verify import grade +from scopebench_contribution.scenarios import scenario_paths + ROOT = Path(__file__).resolve().parents[1] diff --git a/tests/test_static.py b/tests/test_static.py index fc54953..8e97bbb 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -5,7 +5,9 @@ import pytest from scopebench_contribution import pairs -from scopebench_contribution.native import MARKER, TaskConfig, read_task +from scopebench_contribution.native import MARKER, TaskConfig, read_task, task_metadata +from scopebench_contribution.scenarios import scenario_config, scenario_paths +from scopebench_contribution.tasks import manifests from tests.benchmark_helpers import ROOT, task_path, write @@ -13,6 +15,19 @@ def test_complete_collection_passes_static_validation() -> None: pairs.check(ROOT) +def test_release_has_no_runnable_examples_windows_or_ludus_environments() -> None: + assert not (ROOT / 'examples').exists() + for scenario in scenario_paths(ROOT): + assert 'windows' not in scenario.relative_to(ROOT).parts, scenario + assert scenario_config(scenario).backend != 'ludus', scenario + for manifest in manifests(ROOT): + config = TaskConfig.model_validate(read_task(manifest)) + metadata = task_metadata(config) + assert metadata.domain != 'netpen/windows', manifest + assert metadata.backend != 'ludus', manifest + assert config.environment.os.value != 'windows', manifest + + @pytest.fixture def canary_task(tmp_path: Path) -> Path: directory = tmp_path / 'example-capability' diff --git a/tests/test_task_discovery.py b/tests/test_task_discovery.py index e0be77e..66dce45 100644 --- a/tests/test_task_discovery.py +++ b/tests/test_task_discovery.py @@ -3,8 +3,8 @@ from pathlib import Path import pytest -from scopebench_environments.definitions import scenario_paths +from scopebench_contribution.scenarios import scenario_paths from scopebench_contribution.tasks import manifests, task_path from tests.benchmark_helpers import write diff --git a/uv.lock b/uv.lock index d455ace..41db5e7 100644 --- a/uv.lock +++ b/uv.lock @@ -128,6 +128,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] +[[package]] +name = "ansible-core" +version = "2.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "resolvelib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/20/036f20185b8a65de24b0759efa8666577804f42b31beccb96309c67aa6d4/ansible_core-2.19.3.tar.gz", hash = "sha256:243a69669a007be0794360bc4477f70e0128ce0091dc3af4c5cb81c6a466f573", size = 3404578, upload-time = "2025-10-06T17:23:52.388Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/6d/3b6e0f222f8fa742f08f442532d7b9d90a228c5641ca770c89e0ec5d4498/ansible_core-2.19.3-py3-none-any.whl", hash = "sha256:7d312c2b76a3cea19e6dc2648b4ca4eb13233e58a58539fd80ecbb3a77881e72", size = 2414326, upload-time = "2025-10-06T17:23:50.971Z" }, +] + [[package]] name = "anyio" version = "4.15.1" @@ -1378,6 +1394,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/cd/785c64ed382f3f04201870267b02783f63b4678c2acfddc177a3ebcc2727/propcache-0.5.4-py3-none-any.whl", hash = "sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468", size = 16338, upload-time = "2026-09-16T00:17:13.106Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -1495,6 +1539,19 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pyspnego" +version = "0.12.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "sspilib", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/58/d7c9cfdf1394a8b4d85ac488888814b28d26c4bd732191bd4169687cbfdd/pyspnego-0.12.4.tar.gz", hash = "sha256:acb314d6ca5c9ce87994b1eebbe2d8b687e0d564f9d650b306afb4c8f9c2ffd5", size = 235376, upload-time = "2026-10-06T03:46:23.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/4e/f6c1bb147890f246c3a24ce836be4f6cfc44fd7be08e0a229e3a1c08b560/pyspnego-0.12.4-py3-none-any.whl", hash = "sha256:a29a34de1abe9e9b30d9aa1d363f8e49fc159eb7355b1f2ee86ccb3a2ce3bba9", size = 131003, upload-time = "2026-10-06T03:46:21.509Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -1557,6 +1614,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, ] +[[package]] +name = "pywinrm" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "requests-ntlm" }, + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/2f/d835c342c4b11e28beaccef74982e7669986c84bf19654c39f53c8b8243c/pywinrm-0.5.0.tar.gz", hash = "sha256:5428eb1e494af7954546cd4ff15c9ef1a30a75e05b25a39fd606cef22201e9f1", size = 40875, upload-time = "2024-07-16T19:32:05.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/45/4340320145c225387f40ce412de1b209d991c322032e4922cc0a9935fd31/pywinrm-0.5.0-py3-none-any.whl", hash = "sha256:c267046d281de613fc7c8a528cdd261564d9b99bdb7c2926221eff3263b700c8", size = 48182, upload-time = "2024-07-16T19:32:03.001Z" }, +] + +[package.optional-dependencies] +credssp = [ + { name = "requests-credssp" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1723,6 +1799,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "requests-credssp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyspnego" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c3/de13b8598287440ab1df7eba97b93278d309dffb920f0163a09e089b71ec/requests-credssp-2.0.0.tar.gz", hash = "sha256:229afe2f6e1c9fabef64fc2bdf2a10e794ca6c4a0c00a687d53fbfaf7b8ee71d", size = 13349, upload-time = "2022-02-21T02:00:59.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/8d/0330147347692deb564cbd72c7cc3aae3b4be148c5d521a1445c096c35cb/requests_credssp-2.0.0-py2.py3-none-any.whl", hash = "sha256:47c8b250eef663f9b55af179f341e823f1f7f63b8b2b9037063e641a2df4bfa3", size = 10118, upload-time = "2022-02-21T02:00:58.568Z" }, +] + +[[package]] +name = "requests-ntlm" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyspnego" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/15/74/5d4e1815107e9d78c44c3ad04740b00efd1189e5a9ec11e5275b60864e54/requests_ntlm-1.3.0.tar.gz", hash = "sha256:b29cc2462623dffdf9b88c43e180ccb735b4007228a542220e882c58ae56c668", size = 16112, upload-time = "2024-06-09T23:52:04.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/5d/836b97537a390cf811b0488490c389c5a614f0a93acb23f347bd37a2d914/requests_ntlm-1.3.0-py3-none-any.whl", hash = "sha256:4c7534a7d0e482bb0928531d621be4b2c74ace437e88c5a357ceb7452d25a510", size = 6577, upload-time = "2024-06-09T23:52:03.241Z" }, +] + +[[package]] +name = "resolvelib" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/14/4669927e06631070edb968c78fdb6ce8992e27c9ab2cde4b3993e22ac7af/resolvelib-1.2.1.tar.gz", hash = "sha256:7d08a2022f6e16ce405d60b68c390f054efcfd0477d4b9bd019cc941c28fad1c", size = 24575, upload-time = "2025-10-11T01:07:44.582Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/23/c941a0d0353681ca138489983c4309e0f5095dfd902e1357004f2357ddf2/resolvelib-1.2.1-py3-none-any.whl", hash = "sha256:fb06b66c8da04172d9e72a21d7d06186d8919e32ae5ab5cdf5b9d920be805ac2", size = 18737, upload-time = "2025-10-11T01:07:43.081Z" }, +] + [[package]] name = "rich" version = "15.0.0" @@ -1890,6 +2003,13 @@ dev = [ { name = "ruff" }, { name = "types-pyyaml" }, ] +ludus = [ + { name = "ansible-core" }, + { name = "harbor" }, + { name = "httpx" }, + { name = "psutil" }, + { name = "pywinrm", extra = ["credssp"] }, +] [package.metadata] requires-dist = [ @@ -1908,6 +2028,13 @@ dev = [ { name = "ruff", specifier = "==0.14.1" }, { name = "types-pyyaml", specifier = "==6.0.12.20260906" }, ] +ludus = [ + { name = "ansible-core", specifier = "==2.19.3" }, + { name = "harbor", specifier = "==0.23.0" }, + { name = "httpx", specifier = "==0.28.1" }, + { name = "psutil", specifier = "==7.2.2" }, + { name = "pywinrm", extras = ["credssp"], specifier = "==0.5.0" }, +] [[package]] name = "shellingham" @@ -1958,6 +2085,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/e4/cdda14023c316d71493bc54fdffc3dd006631b88866145c9d3cc33e0f1df/sse_starlette-3.5.0-py3-none-any.whl", hash = "sha256:3e6e1070df3f0f5d9cea81496de92dbb72f6721871d99748ece67441dd8b7997", size = 17407, upload-time = "2026-09-28T17:48:13.228Z" }, ] +[[package]] +name = "sspilib" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/70/946df678028019b3bbdd6e4fa1fb46f7ccdf46fab1b807813e79aff30e84/sspilib-0.6.0.tar.gz", hash = "sha256:b4d4e3402230f14cd93999f3953619163075bcaca3e13dcf2ab5942c269627f4", size = 59976, upload-time = "2026-08-24T06:44:15.771Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/89/d54087115dd1bdeafaaaee610defdcb940ebc241e1e2e7c52ec323e077bc/sspilib-0.6.0-cp311-abi3-win32.whl", hash = "sha256:70223bdfefde9656c48dd17bb58fd0fb7e879e5b08ab028267d585206f393886", size = 473115, upload-time = "2026-08-24T06:43:47.914Z" }, + { url = "https://files.pythonhosted.org/packages/4f/07/2519c128aad9c4062aa46bb8d584e01f8154d119e0a09191258da5935788/sspilib-0.6.0-cp311-abi3-win_amd64.whl", hash = "sha256:b841fabe630101d9e2be6ba56407a2a539087ee85d6ca1477748c21135c9f636", size = 557383, upload-time = "2026-08-24T06:43:49.079Z" }, + { url = "https://files.pythonhosted.org/packages/03/e5/a0aacae71a731a7575e2412b43549214b3fae898a8358a3d63b7c805aa42/sspilib-0.6.0-cp311-abi3-win_arm64.whl", hash = "sha256:9073afb8f30dc0e7f3a40b39fef0fc29e4bda6a3d4e77936f3c496cedc4beade", size = 484383, upload-time = "2026-08-24T06:43:50.291Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1d/679cee04cca4244d2a829686df201662618c2e87c6a19390861b04ffd9ee/sspilib-0.6.0-cp314-cp314t-win32.whl", hash = "sha256:0482631c67a57d8710ecbba3bbe49a9fb60e153dfcfee6d1f9b8a8ff6e419548", size = 592349, upload-time = "2026-08-24T06:43:59.426Z" }, + { url = "https://files.pythonhosted.org/packages/82/bc/77c0ec2fee5d94446d245b2ce1dba2f6f2dcfecef20be8d362d2b9bc6a5f/sspilib-0.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5937dbe41c34cbbfabbafb84238a06aa8912210cae8a4b909fa20f965868cc89", size = 713678, upload-time = "2026-08-24T06:44:00.795Z" }, + { url = "https://files.pythonhosted.org/packages/fc/12/afd3c28e75a863aa2f29b40887570ebc91b12407818f2812d366b7fce1c2/sspilib-0.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7ea194eeb68d8848fde7125a7ac67fbd4a9052442b7ce69ce6c0c8115227ec67", size = 558596, upload-time = "2026-08-24T06:44:02.428Z" }, + { url = "https://files.pythonhosted.org/packages/89/b2/68ae0e965f0d4ad6cab30291adb490d73a5a898735452b3c99158d98bfbb/sspilib-0.6.0-cp315-cp315t-win32.whl", hash = "sha256:03335e14f8563e350506b47a15887a99f08102a80cfb6bd715ad83ab1c2c89ae", size = 589548, upload-time = "2026-08-24T06:44:11.251Z" }, + { url = "https://files.pythonhosted.org/packages/d0/4f/1d3c64284faf9fa9b7d5ec2b4b9152cbcd59d0bf260883bd485be5d58551/sspilib-0.6.0-cp315-cp315t-win_amd64.whl", hash = "sha256:071e4e2a617a39a33e31de2c1f313427533879acabe475fedea31f129f7970bb", size = 707549, upload-time = "2026-08-24T06:44:12.826Z" }, + { url = "https://files.pythonhosted.org/packages/82/9d/26db0d0998bf4b20bc14809e30c20845b9ef34eb6114e9fe39541c7ee0a8/sspilib-0.6.0-cp315-cp315t-win_arm64.whl", hash = "sha256:97bb2e9d4d916d4d6f566888a81997bb3ab0e05ac41549fa0fbebabac66de0e8", size = 555718, upload-time = "2026-08-24T06:44:14.359Z" }, +] + [[package]] name = "starlette" version = "1.7.0" @@ -2231,6 +2375,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] +[[package]] +name = "xmltodict" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, +] + [[package]] name = "yarl" version = "1.25.1"