diff --git a/app/routers/controller.py b/app/routers/controller.py index 8c0e67fa..5ed3d4eb 100644 --- a/app/routers/controller.py +++ b/app/routers/controller.py @@ -60,6 +60,10 @@ from apps.utils.docker import compose, instance_logs, l4_ports, observe, volume from apps.utils.docker import updater as updater_spawn from apps.utils.greffon import repository +# Bound at import time, NOT accessed as repository.ComposeShapeError at +# exception time: the route tests patch the whole repository module, and an +# except clause on a Mock attribute raises TypeError mid-handler. +from apps.utils.greffon.repository import ComposeShapeError from apps.utils.nginx import conf logger = logging.getLogger("greffer") @@ -254,6 +258,13 @@ def start_greffon( try: greffon_info = repository.get_greffon_info( compose_file, greffon, l4_bind_host=l4_bind_host) + except ComposeShapeError as exc: + # compose-containment Feature 2: a compose whose shape the pipeline + # cannot honour (unsupported construct, malformed entry) fails with + # the offending services.. path, like ConfigRenderError — + # a diagnosable 422, never an opaque 500. Runs before any port + # allocation or volume write, so nothing half-starts. + raise HTTPException(status_code=422, detail=str(exc)) from exc except l4_ports.L4SamePortConflict as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc except l4_ports.L4PortRangeExhausted as exc: diff --git a/apps/utils/docker/compose.py b/apps/utils/docker/compose.py index 0598683f..09b03153 100644 --- a/apps/utils/docker/compose.py +++ b/apps/utils/docker/compose.py @@ -174,7 +174,13 @@ def create_compose_template_from_greffon(compose, greffon_info): for _,volume in greffon_info['volumes'].items(): for container_name, container in volume['containers'].items(): compose['services'][container_name].setdefault('volumes', []) - compose['services'][container_name]['volumes'].append(f'{volume["value"]}:{container["path"]}') + # A preserved short-syntax mount mode (Feature 2: 'v:/p:ro') rides + # the rebuilt mapping, so a read-only declaration stays read-only. + mode = container.get('mode') + mapping = f'{volume["value"]}:{container["path"]}' + if mode: + mapping = f'{mapping}:{mode}' + compose['services'][container_name]['volumes'].append(mapping) for _,network in greffon_info['networks'].items(): for _, container_name in enumerate(network['containers']): compose['services'][container_name]['networks'].append(network['value']) diff --git a/apps/utils/greffon/compose_shape.py b/apps/utils/greffon/compose_shape.py new file mode 100644 index 00000000..2cf7b4b3 --- /dev/null +++ b/apps/utils/greffon/compose_shape.py @@ -0,0 +1,316 @@ +"""Compose-shape normalization (compose-containment Feature 2). + +A single pass in front of ``create_greffon_info`` that turns every legal +compose spelling of ports / volumes / networks into one dict-or-list shape, +and refuses what the pipeline cannot honour with a structured error naming +the service and the key — never an uncaught ``AttributeError``/ +``ValueError``/``KeyError`` surfaced as a bare 500 (the seven-construct +failure table in the epic's Context). + +The contract with ``create_greffon_info`` after this pass: + +- ``services`` is a dict of dicts (missing -> error); +- every service's ``ports`` is a list of ``{"target": str, "protocol": str|None}``; +- every service's ``volumes`` is a list of ``{"source": str, "target": str}``; +- top-level ``networks`` (if present) is a dict whose keys the services may + reference; every service's ``networks`` is a list of declared names; +- ``ports``/``volumes`` in any other shape (a mapping, a bare string) is + refused — compose itself only accepts lists here, and the legacy dead + ``else`` branches that pretended to handle them referenced unbound locals. + +Extraction equivalence (the catalog regression bar): for the short forms the +catalog uses, the values ``create_greffon_info`` derives — ``port_name``, +protocol, volume source/target — are identical before and after; the shapes +that previously crashed now either normalize to those same values or refuse +with a diagnosis. The rebuild in ``create_compose_template_from_greffon`` +erases and re-derives service ``ports``/``volumes``/``networks`` anyway, so +the rendered compose cannot change for entries that parsed before; a +preserved mount mode (``:ro``) is re-rendered onto the rebuilt mapping, so +a read-only declaration stays read-only. +""" + +from __future__ import annotations + +# Access modes docker documents for the third short-syntax mount field (the +# long-syntax ``mode`` key honours ro/rw). A preserved mode is re-rendered +# onto the rebuilt mapping (Feature 2: a read-only declaration stays +# read-only); an unknown token is far more likely a typo ("vol:/p:banna") +# than a mode we should silently drop, so only the documented set is accepted. +_MOUNT_MODES = frozenset( + {"ro", "rw", "z", "Z", "cached", "delegated", "consistent", "default"} +) + +_PROTOCOLS = frozenset({"tcp", "udp"}) + +# compose's implicit default network: a service may reference it without a +# top-level declaration. +_IMPLICIT_NETWORKS = frozenset({"default"}) + + +class ComposeShapeError(Exception): + """A compose the pipeline cannot honour, with the offending path. + + Carries ``service``/``key``/``value`` as attributes for programmatic use + (the future preflight surfaces them verbatim) and renders as + ``services..: `` so a 422 detail reads like the + YAML path the user must fix. + """ + + def __init__(self, service: str | None, key: str, message: str, value=None): + self.service = service + self.key = key + self.value = value + self.message = message + location = f"services.{service}.{key}" if service else key + suffix = f" (got {value!r})" if value is not None else "" + super().__init__(f"{location}: {message}{suffix}") + + +def _bad(service, key, message, value=None): + return ComposeShapeError(service, key, message, value) + + +def _normalize_port(entry, service: str) -> dict: + """One ``ports`` entry (every legal spelling) -> {"target", "protocol"}.""" + if isinstance(entry, int) and not isinstance(entry, bool): + target, protocol = str(entry), None + elif isinstance(entry, str): + # "8080" | "80:80" | "127.0.0.1:8080:80" | "8080/udp" | "80:80/udp" + # — the container port is the LAST colon-separated segment, matching + # the legacy `port.split(':')[-1]` extraction. + raw = entry + protocol = None + if "/" in raw: + raw, proto = raw.rsplit("/", 1) + protocol = proto.lower() + target = raw.rsplit(":", 1)[-1] + elif isinstance(entry, dict): + # Long syntax: {target, published?, protocol?, mode?}. ``published`` + # (the host port) carries no information downstream — the greffer + # rebuilds every mapping from its own allocation — so it is read only + # to be named in errors. + target = entry.get("target") + protocol = entry.get("protocol") + if "target" not in entry: + raise _bad(service, "ports", "long-syntax port requires 'target'", entry) + else: + raise _bad( + service, + "ports", + f"unsupported ports entry type ({type(entry).__name__})", + entry, + ) + + if isinstance(target, int) and not isinstance(target, bool): + # Long syntax allows an integer target ("target": 80). + target = str(target) + if ( + not isinstance(target, str) + or not target.isdigit() + or not (0 < int(target) < 65536) + ): + if isinstance(target, str) and "-" in target: + hint = "port ranges are not supported; declare each port as its own entry" + elif isinstance(entry, dict): + hint = "long-syntax 'target' must be the container port" + else: + hint = "container port must be an integer" + raise _bad(service, "ports", hint, entry) + if protocol is not None and protocol not in _PROTOCOLS: + raise _bad( + service, "ports", f"protocol must be one of {sorted(_PROTOCOLS)}", entry + ) + return {"target": target, "protocol": protocol} + + +def _normalize_volume(entry, service: str) -> dict: + """One ``volumes`` entry -> {"source", "target"}. Bind mounts and + anonymous volumes are refused: the greffer runs docker-compose inside its + own container against the host daemon, so a host path resolves against + the greffer's filesystem, not the operator's — structurally unsupported, + not merely unsafe.""" + if isinstance(entry, str): + parts = entry.split(":") + if len(parts) == 2: + source, target, mode = parts[0], parts[1], None + elif len(parts) == 3 and parts[2] in _MOUNT_MODES: + source, target, mode = parts[0], parts[1], parts[2] + elif len(parts) == 3: + raise _bad( + service, + "volumes", + f"unknown mount mode (known: {sorted(_MOUNT_MODES)})", + entry, + ) + elif len(parts) == 1: + raise _bad( + service, + "volumes", + "anonymous volumes are not supported; declare a named " + "volume in the top-level volumes block", + entry, + ) + else: + raise _bad(service, "volumes", "expected SOURCE:TARGET[:MODE]", entry) + elif isinstance(entry, dict): + vtype = entry.get("type", "volume") + source = entry.get("source") + target = entry.get("target") + # Long syntax expresses access with `read_only` (a bool), NOT the + # short syntax's `:ro` mode token — reading `mode` alone rendered a + # declared read-only mount writable (codex review). + if entry.get("read_only") is True: + mode = "ro" + else: + mode = entry.get("mode") if entry.get("mode") in ("ro", "rw") else None + # Nested volume options cannot be expressed in the rebuilt short + # syntax: subpath mounts a SUBDIRECTORY (dropping it mounts the whole + # volume), nocopy changes population semantics. Accepting the mount + # while dropping the option deploys a DIFFERENT mount — refuse. + nested = entry.get("volume") + if isinstance(nested, dict) and nested: + raise _bad( + service, + "volumes", + "long-syntax volume options (subpath/nocopy) are not " + "supported by the rebuilt mount; declare the subdirectory " + "as its own top-level volume instead", + entry, + ) + if vtype != "volume": + raise _bad( + service, + "volumes", + f"volume type '{vtype}' is not supported; use a named " + "volume (type: volume)", + entry, + ) + if "target" not in entry: + raise _bad( + service, "volumes", "long-syntax volume requires 'target'", entry + ) + if not source: + raise _bad( + service, + "volumes", + "anonymous volumes are not supported; declare a named " + "volume in the top-level volumes block", + entry, + ) + else: + raise _bad( + service, + "volumes", + f"unsupported volumes entry type ({type(entry).__name__})", + entry, + ) + + if not source or not target: + raise _bad(service, "volumes", "source and target must be non-empty", entry) + if source.startswith(("/", ".", "~")): + raise _bad( + service, + "volumes", + "bind mounts are structurally unsupported on this platform; " + "use a named volume declared in the top-level volumes block", + entry, + ) + if not target.startswith("/"): + raise _bad( + service, "volumes", "container target must be an absolute path", entry + ) + return {"source": source, "target": target, "mode": mode} + + +def _normalize_service_networks(entry, service: str) -> list[str]: + """Service ``networks`` (list of names, or mapping name -> {aliases...}) + -> list of names.""" + if isinstance(entry, str): + names = [entry] + elif isinstance(entry, (list, dict)) and all(isinstance(n, str) for n in entry): + # list of names, or mapping name -> {aliases...} (keys are the names) + names = list(entry) + else: + raise _bad( + service, + "networks", + "expected a list of network names or a mapping of name to aliases", + entry, + ) + # An EMPTY list is legal compose (opt out of the default network); the + # pipeline still places the service on greffon_internal_network, so it + # is a no-op here, not a refusal. + return names + + +def normalize_compose(compose) -> dict: + """Validate + normalize a parsed compose in place; return it. + + Raises ``ComposeShapeError`` on the first problem, naming the service and + key. Idempotent: normalized shapes pass through unchanged. + """ + if not isinstance(compose, dict): + raise _bad(None, "compose", "top level must be a mapping", compose) + services = compose.get("services") + if not isinstance(services, dict) or not services: + raise _bad( + None, + "services", + "compose requires a non-empty 'services' mapping", + services if services is not None else "", + ) + for service in services: + if not isinstance(service, str) or not service: + raise _bad( + None, "services", "service names must be non-empty strings", service + ) + top_networks = compose.get("networks") or {} + if isinstance(top_networks, list): + declared = {n for n in top_networks if isinstance(n, str)} + elif isinstance(top_networks, dict): + declared = set(top_networks) + else: + raise _bad( + None, + "networks", + "top-level networks must be a mapping or a list", + top_networks, + ) + + for name, service_def in services.items(): + if not isinstance(service_def, dict): + raise _bad( + name, "", "a service definition must be a mapping", service_def + ) + for key, normalizer in ( + ("ports", _normalize_port), + ("volumes", _normalize_volume), + ): + entries = service_def.get(key) + if entries is None: + continue + if not isinstance(entries, list): + raise _bad( + name, + key, + f"'{key}' must be a list; a mapping or bare value is " + "not valid compose", + entries, + ) + service_def[key] = [normalizer(e, name) for e in entries] + net_entry = service_def.get("networks") + if net_entry is not None: + names = _normalize_service_networks(net_entry, name) + undeclared = [ + n for n in names if n not in declared and n not in _IMPLICIT_NETWORKS + ] + if undeclared: + raise _bad( + name, + "networks", + f"network(s) not declared in the top-level networks " + f"block: {', '.join(sorted(set(undeclared)))}", + net_entry, + ) + service_def["networks"] = names + return compose diff --git a/apps/utils/greffon/repository.py b/apps/utils/greffon/repository.py index e9f85a8e..9c688110 100644 --- a/apps/utils/greffon/repository.py +++ b/apps/utils/greffon/repository.py @@ -4,6 +4,7 @@ import yaml from apps.utils.os.network import get_free_ports from apps.utils.greffon import sticky_ports +from apps.utils.greffon.compose_shape import ComposeShapeError, normalize_compose from apps.utils.docker import l4_ports logger = logging.getLogger("greffer") @@ -18,14 +19,6 @@ def get_compose_file_from_repository(greffon): return yaml.safe_load(r.text) -def _split_proto(raw): - """'51820/udp' -> ('51820', 'udp'); '8080' -> ('8080', None).""" - if '/' in raw: - port, proto = raw.rsplit('/', 1) - return port, proto.lower() - return raw, None - - # Catalog authors set this label on a service whose app streams responses # (Server-Sent Events, long-poll, chunked) so the generated nginx proxy stops # buffering them. Without it nginx's default response buffering holds the @@ -143,6 +136,13 @@ def get_greffon_info(compose, greffon, l4_bind_host='0.0.0.0'): def create_greffon_info(compose, greffon): + # compose-containment Feature 2: one normalization pass up front. Every + # ports/volumes entry is a dict and every networks entry a declared name + # past this point, so the loops below read shapes instead of splitting + # strings — and the seven legal-syntax constructs that used to surface as + # bare AttributeErrors/ValueErrors/KeyErrors (500s) either normalize or + # raised a ComposeShapeError (422) inside normalize_compose already. + compose = normalize_compose(compose) greffon_path = os.path.join( os.getenv('GREFFON_PATH', '/data'), greffon['id']) internal_network_id = 'greffon_internal_network' @@ -212,41 +212,26 @@ def create_greffon_info(compose, greffon): 'containers': {}, 'files': [] } - for _, network_name in enumerate(compose.get('networks', [])): + top_networks = compose.get('networks') or {} + top_network_names = (list(top_networks) if isinstance(top_networks, dict) + else list(top_networks)) + for network_name in top_network_names: + # 'containers' is a LIST of service names (like the internal + # network's above): the rebuild in create_compose_template_from_ + # greffon iterates it and appends the network to each named service. greffon_info['networks'][network_name] = { 'name': network_name, 'value': network_name, - 'containers': {}, + 'containers': [], 'files': [] } for name, service in compose['services'].items(): - ports = service.get('ports', []) - if type(ports) == list: - for port in ports: - port_splited = port.split(':') - port_container, parsed_proto = _split_proto(port_splited[-1]) - port_name = f'{name}_{port_container}' - manager_port = greffon.get('ports', {}).get(port_name, {}) - greffon_info['ports'].append({ - 'port_container': port_container, - 'container_name': name, - 'port_name': port_name, - 'url': manager_port.get('url'), - # Tier/protocol: manager (catalog) is authoritative; fall - # back to parsing ":/udp" from the compose, then to - # the http/tcp defaults. - 'protocol': manager_port.get('protocol') or parsed_proto or 'tcp', - 'exposure_tier': manager_port.get('exposure_tier', 'http'), - # same_port: publish host P -> container P (not declared - # container port) so the app advertises exactly what it - # binds. Manager-declared (L4 only); default off. - 'same_port': bool(manager_port.get('same_port', False)), - 'streaming': _service_streaming(service), - }) - else: - greffon_info['ports'].setdefault(name, {}) - _, raw_container = port.split(':') - port_container, parsed_proto = _split_proto(raw_container) + # normalize_compose guarantees lists of dict entries past this point; + # the legacy if/else branches for mapping-form ports/volumes were dead + # code referencing unbound locals and are gone. + for port in service.get('ports', []) or []: + port_container = port['target'] + parsed_proto = port.get('protocol') port_name = f'{name}_{port_container}' manager_port = greffon.get('ports', {}).get(port_name, {}) greffon_info['ports'].append({ @@ -254,52 +239,46 @@ def create_greffon_info(compose, greffon): 'container_name': name, 'port_name': port_name, 'url': manager_port.get('url'), + # Tier/protocol: manager (catalog) is authoritative; fall + # back to the compose-declared protocol, then to the + # http/tcp defaults. 'protocol': manager_port.get('protocol') or parsed_proto or 'tcp', 'exposure_tier': manager_port.get('exposure_tier', 'http'), + # same_port: publish host P -> container P (not declared + # container port) so the app advertises exactly what it + # binds. Manager-declared (L4 only); default off. 'same_port': bool(manager_port.get('same_port', False)), 'streaming': _service_streaming(service), }) - volumes = service.get('volumes', []) - if type(volumes) == list: - for volume in service.get('volumes', []): - volume_host, volume_container = volume.split(':') - if volume_host not in greffon_info['volumes']: - # todo should handle multi containers - greffon_info['volumes'][volume_host] = { - 'name': volume_host, - 'value': volume_host, - 'containers': { - name: { - 'path': volume_container - } - }, - 'files': [] - } - else: - greffon_info['volumes'][volume_host]['containers'][name] = { - 'path': volume_container - } - else: - volume_host, volume_container = volume.split(':') + for volume in service.get('volumes', []) or []: + volume_host = volume['source'] + volume_container = volume['target'] + volume_mode = volume.get('mode') if volume_host not in greffon_info['volumes']: + # todo should handle multi containers greffon_info['volumes'][volume_host] = { 'name': volume_host, 'value': volume_host, 'containers': { name: { - 'path': volume_container + 'path': volume_container, + **({'mode': volume_mode} if volume_mode else {}) } }, 'files': [] } else: greffon_info['volumes'][volume_host]['containers'][name] = { - 'path': volume_container + 'path': volume_container, + **({'mode': volume_mode} if volume_mode else {}) } - networks = service.get('networks', []) - if type(networks) == list: - for network in networks: - greffon_info['networks'][network]['containers'].append(name) - else: - greffon_info['networks'][networks]['containers'].append(name) + for network in service.get('networks', []) or []: + # compose's implicit 'default' network is referenceable without + # a top-level declaration; the pipeline maps every service onto + # greffon_internal_network regardless, so 'default' is a no-op + # rather than a registration (registering it would invent a + # network the pipeline never uses). + if network == 'default': + continue + greffon_info['networks'][network]['containers'].append(name) return greffon_info diff --git a/tests/test_compose_shape.py b/tests/test_compose_shape.py new file mode 100644 index 00000000..647f1f3d --- /dev/null +++ b/tests/test_compose_shape.py @@ -0,0 +1,349 @@ +"""Tests for compose-shape normalization (compose-containment Feature 2): +every legal spelling of ports/volumes/networks normalizes to one shape, and +every construct the pipeline cannot honour refuses with a ComposeShapeError +naming the service and key — never an uncaught exception (a bare 500). + +The epic's seven-construct failure table is the spec: each row gets a +support case (normalizes; extraction identical to the legacy string-split +for the shapes that parsed before) and/or a refusal case asserting the +STATUS-equivalent error names the offending service.""" + +from __future__ import annotations + +import pytest + +from apps.utils.greffon import repository +from apps.utils.greffon.compose_shape import ComposeShapeError, normalize_compose + + +def _compose(service=None): + service = service if service is not None else {} + return {"services": {"app": service}} + + +def _normalized(service=None): + return normalize_compose(_compose(service))["services"]["app"] + + +class TestPortsNormalize: + def test_bare_string(self): + assert _normalized({"ports": ["8080"]})["ports"] == [ + {"target": "8080", "protocol": None} + ] + + def test_bare_int(self): + # Epic table row: '8080' as int raised AttributeError on .split. + assert _normalized({"ports": [8080]})["ports"] == [ + {"target": "8080", "protocol": None} + ] + + def test_host_container(self): + assert _normalized({"ports": ["80:80"]})["ports"] == [ + {"target": "80", "protocol": None} + ] + + def test_ip_host_container_udp(self): + assert _normalized({"ports": ["127.0.0.1:8080:80/udp"]})["ports"] == [ + {"target": "80", "protocol": "udp"} + ] + + def test_long_syntax(self): + # Epic table row: the {target, published} mapping raised + # AttributeError on .split. + entry = {"target": 80, "published": 8080, "protocol": "udp"} + assert _normalized({"ports": [entry]})["ports"] == [ + {"target": "80", "protocol": "udp"} + ] + + def test_uppercase_protocol_lowered(self): + assert _normalized({"ports": ["53/UDP"]})["ports"][0]["protocol"] == "udp" + + def test_idempotent(self): + once = _normalized({"ports": ["80:80"]}) + twice = normalize_compose({"services": {"app": {"ports": once["ports"]}}}) + assert twice["services"]["app"]["ports"] == once["ports"] + + +class TestPortsRefuse: + def test_non_numeric_target(self): + with pytest.raises(ComposeShapeError) as e: + _normalized({"ports": ["http"]}) + assert "services.app.ports" in str(e.value) + + def test_long_syntax_missing_target(self): + with pytest.raises(ComposeShapeError) as e: + _normalized({"ports": [{"published": 8080}]}) + assert "target" in str(e.value) + + def test_unknown_protocol(self): + with pytest.raises(ComposeShapeError) as e: + _normalized({"ports": ["53/sctp"]}) + assert "protocol" in str(e.value) + + def test_mapping_form_refused(self): + # The legacy dead else-branch shape: invalid compose, now refused + # instead of referencing an unbound local. + with pytest.raises(ComposeShapeError) as e: + _normalized({"ports": {"80": "80"}}) + assert isinstance(e.value.value, dict) + + +class TestVolumesNormalize: + def test_two_part(self): + assert _normalized({"volumes": ["db_data:/var/lib/postgresql"]})["volumes"] == [ + {"source": "db_data", "target": "/var/lib/postgresql", "mode": None} + ] + + def test_three_part_ro(self): + # Epic table row: 'vol:/path:ro' raised ValueError on unpack. The + # mode is PRESERVED (a dropped :ro would deploy read-only mounts + # read-write). + assert _normalized({"volumes": ["db_data:/data:ro"]})["volumes"] == [ + {"source": "db_data", "target": "/data", "mode": "ro"} + ] + + def test_long_syntax(self): + # Epic table row: {type, source, target} raised AttributeError. + entry = {"type": "volume", "source": "db_data", "target": "/data"} + assert _normalized({"volumes": [entry]})["volumes"] == [ + {"source": "db_data", "target": "/data", "mode": None} + ] + + def test_long_syntax_mode_honoured(self): + entry = {"type": "volume", "source": "db", "target": "/d", "mode": "ro"} + assert _normalized({"volumes": [entry]})["volumes"][0]["mode"] == "ro" + + def test_long_syntax_read_only_renders_ro(self): + # Codex review: long syntax expresses access with read_only, not the + # short syntax's mode token — ignoring it deployed a declared + # read-only mount WRITABLE. The rendered mapping must stay :ro. + from apps.utils.docker.compose import create_compose_template_from_greffon + + compose = {"services": {"app": {"image": "x", "volumes": [ + {"type": "volume", "source": "db", "target": "/d", + "read_only": True}]}}} + info = repository.create_greffon_info(compose, _greffon()) + rendered = create_compose_template_from_greffon(compose, info) + mounts = rendered["services"]["app"]["volumes"] + assert any(m.endswith(":ro") for m in mounts), mounts + + def test_long_syntax_nested_options_refused(self): + # subpath/nocopy cannot be expressed in the rebuilt short syntax; + # dropping them would deploy a DIFFERENT mount, so refuse. + with pytest.raises(ComposeShapeError, match="subpath/nocopy"): + _normalized({"volumes": [ + {"type": "volume", "source": "db", "target": "/d", + "volume": {"subpath": "tenant1"}}]}) + with pytest.raises(ComposeShapeError, match="subpath/nocopy"): + _normalized({"volumes": [ + {"type": "volume", "source": "db", "target": "/d", + "volume": {"nocopy": True}}]}) + + def test_mode_preserved_into_render(self): + # The end-to-end bar: a read-only declaration must stay read-only + # through the rebuild (the mapping is re-rendered, not copied). + from apps.utils.docker.compose import create_compose_template_from_greffon + + compose = { + "services": {"app": {"image": "x", "volumes": ["db_data:/data:ro"]}} + } + info = repository.create_greffon_info(compose, _greffon()) + rendered = create_compose_template_from_greffon(compose, info) + mounts = rendered["services"]["app"]["volumes"] + assert any(m.endswith(":ro") for m in mounts), mounts + + def test_modeless_mount_renders_without_suffix(self): + from apps.utils.docker.compose import create_compose_template_from_greffon + + compose = {"services": {"app": {"image": "x", "volumes": ["db:/data"]}}} + info = repository.create_greffon_info(compose, _greffon()) + rendered = create_compose_template_from_greffon(compose, info) + mounts = rendered["services"]["app"]["volumes"] + assert all(not m.rsplit(":", 1)[-1] in ("ro", "rw") for m in mounts), mounts + + +class TestVolumesRefuse: + def test_unknown_mode(self): + with pytest.raises(ComposeShapeError) as e: + _normalized({"volumes": ["db_data:/data:banna"]}) + assert "mount mode" in str(e.value) + + def test_four_part(self): + with pytest.raises(ComposeShapeError): + _normalized({"volumes": ["a:b:c:d"]}) + + def test_bind_short(self): + with pytest.raises(ComposeShapeError) as e: + _normalized({"volumes": ["/host/path:/data"]}) + assert "bind mounts are structurally unsupported" in str(e.value) + + def test_bind_long(self): + with pytest.raises(ComposeShapeError) as e: + _normalized( + {"volumes": [{"type": "bind", "source": "/host", "target": "/data"}]} + ) + assert "bind" in str(e.value) + + def test_tmpfs_long(self): + with pytest.raises(ComposeShapeError): + _normalized({"volumes": [{"type": "tmpfs", "target": "/tmp"}]}) + + def test_anonymous(self): + with pytest.raises(ComposeShapeError) as e: + _normalized({"volumes": ["/data"]}) + assert "anonymous" in str(e.value) + + def test_relative_target(self): + with pytest.raises(ComposeShapeError): + _normalized({"volumes": ["db_data:data"]}) + + +class TestNetworks: + def test_service_list_form(self): + compose = { + "networks": {"backend": {}}, + "services": {"app": {"networks": ["backend"]}}, + } + normalize_compose(compose) + assert compose["services"]["app"]["networks"] == ["backend"] + + def test_service_dict_form(self): + # Epic table row: mapping form raised TypeError unhashable 'dict'. + compose = { + "networks": {"backend": {}}, + "services": {"app": {"networks": {"backend": {"aliases": ["app-1"]}}}}, + } + normalize_compose(compose) + assert compose["services"]["app"]["networks"] == ["backend"] + + def test_top_level_list_form(self): + compose = { + "networks": ["backend"], + "services": {"app": {"networks": ["backend"]}}, + } + normalize_compose(compose) + assert compose["services"]["app"]["networks"] == ["backend"] + + def test_implicit_default_end_to_end(self): + # compose's implicit 'default' network is referenceable without a + # top-level declaration. The normalizer allows it, so the CONSUMER + # must tolerate it: create_greffon_info must not KeyError on the + # unregistered name. (The vacuous normalize-only version of this + # test once passed while the route 500'd — adversarial-review + # finding.) 'default' is a membership no-op: the service rides the + # internal network like every other, and nothing named 'default' + # is invented. + compose = {"services": {"app": {"networks": ["default"]}}} + info = repository.create_greffon_info(compose, _greffon()) + assert "default" not in info["networks"] + assert ( + "app" in info["networks"]["greffon_internal_network"]["containers"] + ) + + def test_implicit_default_string_form_end_to_end(self): + compose = {"services": {"app": {"networks": "default"}}} + info = repository.create_greffon_info(compose, _greffon()) + assert "default" not in info["networks"] + + def test_default_alongside_declared_network_end_to_end(self): + compose = { + "networks": {"backend": {}}, + "services": {"app": {"networks": ["default", "backend"]}}, + } + info = repository.create_greffon_info(compose, _greffon()) + assert info["networks"]["backend"]["containers"] == ["app"] + assert "default" not in info["networks"] + + def test_empty_networks_list_is_noop(self): + # networks: [] is legal compose (opt out of the default network) and + # the pipeline still places the service on the internal network, so + # it normalizes as a no-op instead of refusing (a refusal would + # regress a spelling the legacy pipeline deployed fine). + assert _normalized({"networks": []})["networks"] == [] + compose = {"services": {"app": {"networks": []}}} + info = repository.create_greffon_info(compose, _greffon()) + assert ( + "app" in info["networks"]["greffon_internal_network"]["containers"] + ) + + def test_port_range_message(self): + with pytest.raises(ComposeShapeError) as e: + _normalized({"ports": ["3000-3005:3000-3005"]}) + assert "port ranges are not supported" in str(e.value) + + def test_undeclared_refused(self): + # Previously a KeyError (500); now a named refusal. + compose = {"services": {"app": {"networks": ["ghost"]}}} + with pytest.raises(ComposeShapeError) as e: + normalize_compose(compose) + assert "ghost" in str(e.value) + assert "services.app.networks" in str(e.value) + + def test_end_to_end_greffon_info(self): + # Epic table row: a top-level networks block used by a service raised + # AttributeError ('dict' has no attribute 'append'). The full + # create_greffon_info path now lands the service on the network with + # containers as a LIST (the shape the rebuild iterates). + compose = { + "networks": {"backend": {}}, + "services": { + "app": {"image": "x", "networks": ["backend"]}, + "db": {"image": "y"}, + }, + } + info = repository.create_greffon_info(compose, _greffon()) + assert info["networks"]["backend"]["containers"] == ["app"] + + +class TestTopLevel: + def test_missing_services(self): + # Epic table row: KeyError; now a named refusal. + with pytest.raises(ComposeShapeError) as e: + normalize_compose({"version": "3"}) + assert e.value.key == "services" + assert "services" in str(e.value) + + def test_non_mapping_compose(self): + with pytest.raises(ComposeShapeError): + normalize_compose(["services"]) + + def test_service_not_mapping(self): + with pytest.raises(ComposeShapeError) as e: + normalize_compose({"services": {"app": "nginx"}}) + assert "services.app" in str(e.value) + + +def _greffon(): + return { + "id": "i1", + "configurations": [], + "ports": {}, + "cert": {"certificate": "", "private_key": ""}, + } + + +# The catalog regression bar, at unit scope: for the short forms the catalog +# uses, create_greffon_info derives identical port/volume values through the +# normalizer as the legacy string-split did. +_EQUIVALENCE_CASES = [ + ( + {"ports": ["80:80"], "volumes": ["db:/data"]}, + ("app_80", "tcp", "db", "/data"), + ), + ({"ports": [8080]}, ("app_8080", "tcp", None, None)), + ({"ports": ["53/udp"]}, ("app_53", "udp", None, None)), + ({"ports": ["127.0.0.1:9000:9000"]}, ("app_9000", "tcp", None, None)), +] + + +class TestExtractionEquivalence: + @pytest.mark.parametrize("service,expected", _EQUIVALENCE_CASES) + def test_extraction(self, service, expected): + port_name, proto, vol, target = expected + info = repository.create_greffon_info(_compose(service), _greffon()) + port = info["ports"][0] + assert (port["port_name"], port["protocol"]) == (port_name, proto) + if vol is not None: + assert info["volumes"][vol]["containers"] == {"app": {"path": target}} + else: + assert set(info["volumes"]) == {"greffon_nginx"} diff --git a/tests/test_controller_routes.py b/tests/test_controller_routes.py index a055c44a..8e99f9df 100644 --- a/tests/test_controller_routes.py +++ b/tests/test_controller_routes.py @@ -158,6 +158,31 @@ async def test_start_same_port_conflict_returns_409(client: AsyncClient) -> None assert "l4_same_port_conflict" in r.json()["detail"] +@pytest.mark.asyncio +async def test_start_compose_shape_error_returns_422( + client: AsyncClient, +) -> None: + # compose-containment Feature 2: an unnormalizable compose is a 422 whose + # detail names the offending services.. path — never the bare + # 500 the seven uncaught-exception rows used to produce. The exception + # class is caught by its import-time name, so the whole-module patch here + # exercises exactly the binding the route relies on. + from apps.utils.greffon.compose_shape import ComposeShapeError + + with patch("app.routers.controller.repository") as mock_repo: + mock_repo.get_compose_file_from_repository.return_value = {} + mock_repo.get_greffon_info.side_effect = ComposeShapeError( + "app", "ports", "container port must be an integer", "http") + r = await client.post( + "/api/controller/start/", + json=SAMPLE_START_PAYLOAD, + headers={TOKEN_HEADER: "test-token"}, + ) + assert r.status_code == 422 + assert "services.app.ports" in r.json()["detail"] + assert "container port must be an integer" in r.json()["detail"] + + @pytest.mark.asyncio async def test_start_range_exhausted_returns_409(client: AsyncClient) -> None: from apps.utils.docker.l4_ports import L4PortRangeExhausted diff --git a/tests/test_l4_network_exposure.py b/tests/test_l4_network_exposure.py index 962ea0a2..2d276198 100644 --- a/tests/test_l4_network_exposure.py +++ b/tests/test_l4_network_exposure.py @@ -5,8 +5,11 @@ owning service. Coverage spans the four touched modules: - apps/utils/os/network.py — UDP-aware get_free_ports - - apps/utils/greffon/repository.py — _split_proto, protocol/tier resolution, - per-protocol host-port allocation + - apps/utils/greffon/compose_shape.py — port protocol parsing (the behavior + the old repository._split_proto had), + plus create_greffon_info tier + resolution, per-protocol host-port + allocation - apps/utils/docker/compose.py — service-published L4 mappings, nginx-sidecar L4 exclusion - apps/utils/nginx/conf.py — http_ports filtering (no L4 server block) @@ -121,27 +124,35 @@ def _spy(family, type_, *args, **kwargs): # --------------------------------------------------------------------------- -# 2. _split_proto + create_greffon_info protocol / tier resolution +# 2. compose_shape protocol parsing + create_greffon_info tier resolution # --------------------------------------------------------------------------- class SplitProtoTests(TestCase): - """Tests for apps.utils.greffon.repository._split_proto.""" + """Protocol parsing, at its home since compose-containment Feature 2: + apps.utils.greffon.compose_shape._normalize_port (the behavior the old + the old repository._split_proto implemented; the behavior moved to + compose_shape._normalize_port unchanged).""" + + def _port(self, spec): + from apps.utils.greffon.compose_shape import normalize_compose + compose = {'services': {'app': {'ports': [spec]}}} + return normalize_compose(compose)['services']['app']['ports'][0] def test_split_udp(self): - from apps.utils.greffon.repository import _split_proto - self.assertEqual(_split_proto('51820/udp'), ('51820', 'udp')) + self.assertEqual(self._port('51820/udp'), + {'target': '51820', 'protocol': 'udp'}) def test_split_tcp_explicit(self): - from apps.utils.greffon.repository import _split_proto - self.assertEqual(_split_proto('8080/tcp'), ('8080', 'tcp')) + self.assertEqual(self._port('8080/tcp'), + {'target': '8080', 'protocol': 'tcp'}) def test_split_no_proto(self): - from apps.utils.greffon.repository import _split_proto - self.assertEqual(_split_proto('8080'), ('8080', None)) + self.assertEqual(self._port('8080'), + {'target': '8080', 'protocol': None}) def test_split_uppercase_proto_is_lowered(self): - from apps.utils.greffon.repository import _split_proto - self.assertEqual(_split_proto('51820/UDP'), ('51820', 'udp')) + self.assertEqual(self._port('51820/UDP'), + {'target': '51820', 'protocol': 'udp'}) class CreateGreffonInfoL4Tests(TestCase):