[FEATURE] Add an extension mechanism with discovery and codegen support

AUTHORING.md

@@ -169,8 +169,9 @@ lets your own models slot in alongside Overture's — see
 
 The library is designed to support data producer extensions through multiple patterns.
 This extensibility is a core feature that allows organizations to add custom fields and
-types while maintaining compatibility with the base Overture schema. We are in the
-process of determining how this should work.
+types while maintaining compatibility with the base Overture schema — registering whole
+models is covered just below, and attaching fields to models you do not own is covered
+in [Model Extensions](#model-extensions).
 
 ### Model Registration via Entry Points
 
@@ -213,10 +214,10 @@ to filter the working set without importing every model:
 from overture.schema.system.discovery import (
     TagSelector,
     discover_models,
-    filter_models,
+    select_models,
 )
 
-models = discover_models()
+models = discover_models(apply_extensions=False)
 # {
 #   ModelKey(name="building", entry_point="overture.schema.buildings:Building",
 #            tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building,
@@ -225,7 +226,7 @@ models = discover_models()
 #   ...
 # }
 
-buildings = filter_models(
+buildings = select_models(
     models,
     TagSelector(include_any=("overture:theme=buildings",)),
 )
@@ -238,6 +239,67 @@ to attach custom tags during discovery. See the [`overture-schema-system`
 README](packages/overture-schema-system/README.md#tagging) for tag format,
 reserved namespaces, and provider authoring.
 
+### Model Extensions
+
+A data producer can attach optional fields to models it does not own. An
+extension is registered on the same `overture.models` entry-point group as any
+other model; discovery recognizes it as an extension by its `Extends` metadata.
+A model extension declares its targets with `@extends`:
+
+```python
+from overture.schema.system.extension import extends
+
+
+@extends(Place)
+class OperatingHours(BaseModel):
+    primary: list[str]
+```
+
+Non-model extensions (e.g. a scalar `NewType`) use `Extends(...)` inside
+`Annotated` metadata instead. During discovery each extension is exposed as a
+standalone one-field wrapper model -- internal merge machinery, hidden by
+`select_models` unless `include_extension_entries=True` is passed -- and the
+extension pass adds the field, optional and named after the entry point, to
+every registered model the targets resolve to. `select_models` applies that
+pass after selection, so excluding the `extension` tag (or an extension's more
+specific tag) opts out of the merge itself, not just the wrapper's visibility.
+
+#### How Targets Resolve
+
+A target may be a model class or a type expression resolving to model classes:
+unions, `Annotated`, `NewType`, and `RootModel`. Two rules govern resolution:
+
+- A union qualifies only if *every* arm resolves to models -- `Place | int` is
+  rejected as a target.
+- A `RootModel` subclass is never a model leaf itself, even though it is a
+  `BaseModel` subclass. It is an alias for its root annotation, and resolution
+  recurses into the root -- at any nesting depth.
+
+The second rule cuts both ways: a `RootModel` over models is an alias for its
+arms, while a `RootModel` over a scalar resolves to no model at all, even when
+nested inside an otherwise valid expression:
+
+```python
+class Segment(RootModel[RoadSegment | RailSegment]):
+    pass
+
+
+class Version(RootModel[int]):
+    pass
+
+
+Extends(Segment)  # OK -- extends RoadSegment and RailSegment
+Extends(Version)  # TypeError -- scalar root resolves to no models
+Extends(Segment | Version)  # TypeError -- every union arm must resolve
+```
+
+The extension pass applies the same alias view to registered entries: a
+registered `Segment` is rebuilt as a subclass whose root annotation carries the
+extended arms, while a registered `Version` passes through unchanged, since its
+scalar root contains nothing to extend. Container types (`list[Place]`,
+`dict[str, Place]`) are opaque on both sides: models nested inside them are
+neither valid targets nor rewritten. A self-referential root has no finite
+shape and is rejected.
 
 ---
 

SCHEMA_GUIDE.md

@@ -719,17 +719,20 @@ A `ModelKey` carries `.name`, `.entry_point` (`"module:Class"`), and `.tags`
 (a `frozenset[str]`). Filter the same way the CLI does:
 
 ```python
-from overture.schema.system.discovery import TagSelector, discover_models, filter_models
+from overture.schema.system.discovery import TagSelector, discover_models, select_models
 
-models = discover_models()
+models = discover_models(apply_extensions=False)
 
-buildings = filter_models(
+buildings = select_models(
     models, TagSelector(include_any=("overture:theme=buildings",))
 )
 ```
 
 `TagSelector` takes `include_any` (OR scope), `require_all` (AND narrowing), and
-`exclude_any` (OR-NOT). An empty selector returns the input unchanged.
+`exclude_any` (OR-NOT). An empty selector selects everything. `select_models` expects
+the raw registry (`apply_extensions=False`) and applies the surviving extensions after
+selection; standalone extension wrapper entries stay hidden unless
+`include_extension_entries=True` is passed.
 
 ### 2.7 Reading enum member documentation
 
@@ -1727,15 +1730,17 @@ covers new themes and third-party extensions.
 
 ```python
 import click
-from overture.schema.system.discovery import discover_models, filter_models, TagSelector
+from overture.schema.system.discovery import discover_models, select_models, TagSelector
 from overture.schema.cli.tag_options import tag_selection_options, build_selector
 
 
 @click.command()
 @tag_selection_options  # gives you --tag / --filter / --exclude for free
 def report(tags, filters, excludes):
     """Report field counts for the selected feature types."""
-    models = filter_models(discover_models(), build_selector(tags, filters, excludes))
+    models = select_models(
+        discover_models(apply_extensions=False), build_selector(tags, filters, excludes)
+    )
     for key, model in sorted(models.items(), key=lambda kv: kv[0].name):
         n = len(model.model_fields) if hasattr(model, "model_fields") else "—"
         click.echo(f"{key.name:20} {n}")

packages/overture-schema-cli/changelog.d/634.feature.md

@@ -0,0 +1 @@
+Listed standalone extension entries in `list-types` while keeping their permissive wrapper models out of validation entirely; extension data validates through the feature models it extends, and `--exclude extension` opts out of extension fields.

packages/overture-schema-cli/src/overture/schema/cli/commands.py

@@ -23,7 +23,7 @@
     ModelKey,
     TagSelector,
     discover_models,
-    filter_models,
+    select_models,
 )
 from overture.schema.system.discovery.tag import get_values_for_key
 from overture.schema.system.feature import Feature
@@ -35,7 +35,10 @@
     group_errors_by_discriminator,
     select_most_likely_errors,
 )
-from .tag_options import build_selector, tag_selection_options
+from .tag_options import (
+    build_selector,
+    tag_selection_options,
+)
 from .type_analysis import StructuralTuple, get_item_index, introspect_union
 from .types import ErrorLocation, UnionType, ValidationErrorDict
 
@@ -201,9 +204,18 @@ def resolve_types(
     *,
     type_names: tuple[str, ...] = (),
 ) -> UnionType:
-    """Resolve a TagSelector + type-names into a Pydantic union type."""
-    models = discover_models()
-    models = filter_models(models, selector, type_names=type_names)
+    """Resolve a TagSelector + type-names into a Pydantic union type.
+
+    Discovers the raw registry and lets `select_models` apply extensions after
+    selection, so the selector's excludes opt out of extension fields
+    (``--exclude extension`` yields the un-extended models) while includes and
+    type names only narrow which models are validated against. Standalone
+    extension wrapper models are internal merge machinery and never resolve
+    to a validatable type; the extension *fields* they contribute remain
+    available on the feature models they target.
+    """
+    models = discover_models(apply_extensions=False)
+    models = select_models(models, selector, type_names=type_names)
 
     if not models:
         raise ValueError("No models found matching the specified criteria")
@@ -880,8 +892,14 @@ def list_types(
       $ overture-schema list-types --group-by overture:theme
     """
     try:
-        models = discover_models()
-        models = filter_models(models, build_selector(tags, filters, excludes))
+        models = discover_models(apply_extensions=False)
+        # A listing is introspection, not selection: show every discoverable
+        # entry, extension wrappers included.
+        models = select_models(
+            models,
+            build_selector(tags, filters, excludes),
+            include_extension_entries=True,
+        )
 
         if group_by:
             grouped_models: dict[str, set[ModelKey]] = {}

packages/overture-schema-cli/tests/test_resolve_types.py

@@ -1,6 +1,6 @@
-"""Tests for resolve_types — CLI glue between filter_models and union creation.
+"""Tests for resolve_types — CLI glue between select_models and union creation.
 
-The combinator algebra of filter_models itself is covered in
+The selector combinator algebra itself is covered in
 `test_discovery_filter_models.py` in the system package.
 """
 
@@ -9,9 +9,11 @@
 from unittest.mock import patch
 
 import pytest
+from pydantic import BaseModel
 
 from overture.schema.cli.commands import resolve_types
 from overture.schema.system.discovery import ModelKey, TagSelector
+from overture.schema.system.extension import extends, wrap_extension
 
 DISCOVER_MODELS = "overture.schema.cli.commands.discover_models"
 
@@ -80,3 +82,57 @@ def test_type_names_are_case_sensitive() -> None:
     # Uppercase doesn't.
     with pytest.raises(ValueError, match="No models found"):
         resolve_types(TagSelector(), type_names=("BUILDING",))
+
+
+# ---------------------------------------------------------------------------
+# Extension application at resolution time
+# ---------------------------------------------------------------------------
+
+
+class Diner(BaseModel):
+    name: str
+
+
+@extends(Diner)
+class OpeningHours(BaseModel):
+    primary: str | None = None
+
+
+_maybe_wrapper = wrap_extension("opening_hours", OpeningHours)
+assert _maybe_wrapper is not None
+
+DINER_KEY = ModelKey(
+    name="diner", entry_point="mock:Diner", tags=frozenset({"feature"})
+)
+WRAPPER_KEY = ModelKey(
+    name="opening_hours",
+    entry_point="mock:OpeningHours",
+    tags=frozenset({"extension"}),
+)
+EXTENSION_MODELS = {DINER_KEY: Diner, WRAPPER_KEY: _maybe_wrapper}
+
+
+class TestExtensionApplication:
+    """Extensions apply at selection time, so the selector's excludes opt out.
+
+    Pins the load-bearing ordering: were extensions merged before the selector
+    ran, `--exclude extension` would remove the wrapper entry but leave the
+    merged field on the feature models.
+    """
+
+    def test_default_resolution_merges_extension_fields(self) -> None:
+        with patch(DISCOVER_MODELS, return_value=EXTENSION_MODELS):
+            resolved = resolve_types(TagSelector())
+        assert isinstance(resolved, type) and issubclass(resolved, Diner)
+        assert "opening_hours" in resolved.model_fields
+
+    def test_exclude_extension_yields_unextended_models(self) -> None:
+        with patch(DISCOVER_MODELS, return_value=EXTENSION_MODELS):
+            resolved = resolve_types(TagSelector(exclude_any=("extension",)))
+        assert resolved is Diner
+
+    def test_type_name_narrowing_keeps_extension_fields(self) -> None:
+        with patch(DISCOVER_MODELS, return_value=EXTENSION_MODELS):
+            resolved = resolve_types(TagSelector(), type_names=("diner",))
+        assert isinstance(resolved, type) and issubclass(resolved, Diner)
+        assert "opening_hours" in resolved.model_fields

packages/overture-schema-codegen/changelog.d/634.feature.md

@@ -0,0 +1 @@
+Added extension-field provenance to extraction (`is_extension`) and rendered an *(extension)* tag on extension-contributed fields in generated markdown.

packages/overture-schema-codegen/src/overture/schema/codegen/cli.py

@@ -7,10 +7,13 @@
 
 import click
 
-from overture.schema.cli.tag_options import build_selector, tag_selection_options
+from overture.schema.cli.tag_options import (
+    build_selector,
+    tag_selection_options,
+)
 from overture.schema.system.discovery import (
     discover_models,
-    filter_models,
+    select_models,
     split_entry_point,
 )
 
@@ -59,7 +62,11 @@ def cli() -> None:
 @cli.command("list")
 def list_models() -> None:
     """List all discovered models."""
-    models = discover_models()
+    # A listing is introspection, not selection: show every discoverable
+    # entry, extension wrappers included.
+    models = select_models(
+        discover_models(apply_extensions=False), include_extension_entries=True
+    )
     # Name every entry from its entry point, not the loaded object: a
     # discriminated union loads as an `Annotated[...]` alias with no
     # `__name__`, so `str(model)` would print the whole type expression.
@@ -106,9 +113,9 @@ def generate(
     if output_format != "pyspark" and test_output_dir is not None:
         raise click.UsageError("--test-output-dir is only valid with --format pyspark")
 
-    all_models = discover_models()
+    all_models = discover_models(apply_extensions=False)
 
-    models = filter_models(all_models, build_selector(tags, filters, excludes))
+    models = select_models(all_models, build_selector(tags, filters, excludes))
 
     if output_dir:
         output_dir.mkdir(parents=True, exist_ok=True)

packages/overture-schema-codegen/src/overture/schema/codegen/extraction/model_extraction.py

@@ -8,6 +8,7 @@
 from pydantic.fields import FieldInfo
 from pydantic_core import PydanticUndefined
 
+from overture.schema.system.extension import applied_extensions
 from overture.schema.system.model_constraint import ModelConstraint
 
 from .docstring import clean_docstring
@@ -159,6 +160,7 @@ def _extract_model_recursive(
     descendant_ancestors = ancestors | {model_class}
 
     model_resolver, union_resolver = _make_resolvers(cache, descendant_ancestors)
+    extensions = applied_extensions(model_class)
 
     fields: list[FieldSpec] = []
     for field_name in _field_order(model_class):
@@ -185,6 +187,7 @@ def _extract_model_recursive(
                 description=field_info.description or ti_description,
                 is_required=_is_field_required(field_info, is_optional),
                 is_optional=is_optional,
+                is_extension=field_name in extensions,
             )
         )
 

packages/overture-schema-codegen/src/overture/schema/codegen/extraction/specs.py

@@ -134,6 +134,7 @@ class FieldSpec:
     description: str | None = None
     is_required: bool = True
     is_optional: bool = False
+    is_extension: bool = False
 
 
 @dataclass