Fix optional collection and nullable enum validation

.agents/skills/loom-framework/SKILL.md

@@ -75,6 +75,24 @@ consumer validation.
 
 ## Architecture Boundaries
 
+- `expr.IsNullable` supplies effective nullability to prepared JSON schema
+  matching and lowered `JSONShape` descriptions. Nullable null bypasses value
+  predicates, including enums, while ordinary Any null still obeys its enum.
+  Inline JSON Schema and OpenAPI lift named occurrence refinements inside the
+  nullable value domain. Runtime descriptors validate before matching; null
+  never hides malformed descriptions or changes untagged union uniqueness.
+  Keep the nullable-enum model beside `pkg/tla/untagged_json` and schema/codec
+  agreement tests aligned.
+
+- Shared `recurseValidationCode` guards optional native array/map occurrences
+  before their predicates. Nil is absence only there; required roots, elements
+  and extracted Optional/Nullable values retain their concrete-value validation.
+  `AttributeContext.CollectionNilIsEmpty` records protobuf's nil-as-empty
+  storage independently of scalar pointers; context copies retain it so wrapped
+  and streamed collection predicates still run. Keep named occurrence guards and
+  the collection-presence model under
+  `codegen/tla/validation_carrier` aligned; do not patch individual transports.
+
 - `execution/` owns typed activity names, options, registration/execution
   contracts, and futures. `execution/adapter` owns concrete-handler reflection;
   it does not own serialization. `execution/selvedge` uses native generic

.agents/skills/loom/SKILL.md

@@ -394,6 +394,11 @@ For maps, `MinLength` and `MaxLength` count entries and export as
 `minProperties` and `maxProperties`, including named map types. Regenerate
 contracts after upgrading so schema validators enforce the declared map bounds.
 
+Optional array/map fields may be omitted despite `MinLength`. Nil native
+collections represent absence; nonnil empty collections must satisfy the bounds.
+Required fields still need presence and valid contents. Regenerate services and
+transports after upgrading to adopt the optional collection validation fix.
+
 Unsigned integer schemas use `uint32` for `UInt32` and `uint64` for `UInt64`
 and `UInt`. Regenerate after upgrading and review generated clients. These
 format annotations describe the domain; validator format support is optional.
@@ -818,6 +823,9 @@ Correct the key design and regenerate instead of editing generated declarations.
   `encoding/json/jsontext.Value`. Do not use the legacy raw-message type.
 - Treat requiredness and nullability as independent. `Required("field")`
   requires presence; `Nullable()` allows a present field to contain JSON null.
+- `Nullable()` plus `Enum(...)` admits null and restricts concrete values to the
+  enum. Ordinary validators and untagged union codecs follow the same rule,
+  including inherited nullability. Upgrade Loom and regenerate to adopt the fix.
 - Generated nullable fields use `loom.Nullable[T]`. Its zero value is absent;
   use `loom.NullValue[T]()` for explicit null and `loom.NullableValue(value)`
   for a concrete value. Inspect it with `Present`, `IsNull`, and `Value`.

codegen/generator/generation_determinism_test.go

@@ -14,6 +14,7 @@ import (
 	servicetestdata "github.com/CaliLuke/loom/codegen/service/testdata"
 	"github.com/CaliLuke/loom/eval"
 	grpctestdata "github.com/CaliLuke/loom/grpc/codegen/testdata"
+	httptestdata "github.com/CaliLuke/loom/http/codegen/testdata"
 )
 
 const generationDeterminismHelperPath = "LOOM_GENERATION_DETERMINISM_HELPER_PATH"
@@ -28,6 +29,8 @@ var generationDeterminismDSLs = []struct {
 	{"PkgPathMultipleDSL", servicetestdata.PkgPathMultipleDSL},
 	{"StructMetaTypeDSL", grpctestdata.StructMetaTypeDSL},
 	{"PayloadWithNestedTypesDSL", grpctestdata.PayloadWithNestedTypesDSL},
+	{"OptionalCollectionValidationDSL", optionalCollectionValidationDSL},
+	{"UntaggedArraysDSL", httptestdata.UntaggedArraysDSL},
 }
 
 // TestGeneratedFilesAreByteStableAcrossIndependentProcesses generates the

codegen/generator/optional_collection_validation_test.go

@@ -0,0 +1,150 @@
+package generator
+
+import (
+	"testing"
+
+	dsl "github.com/CaliLuke/loom/dsl"
+)
+
+func TestOptionalCollectionValidationGenerated(t *testing.T) {
+	runDesignHarness(t, "example.com/collections", optionalCollectionValidationDSL, optionalCollectionValidationHarness)
+}
+
+func optionalCollectionValidationDSL() {
+	dsl.API("collections", func() {
+		dsl.JSONRPC(func() {})
+	})
+	bounds := func() {
+		dsl.MinLength(2)
+		dsl.MaxLength(4)
+	}
+	list := dsl.Type("List", dsl.ArrayOf(dsl.String), bounds)
+	settings := dsl.Type("Settings", func() {
+		dsl.Field(1, "levels", dsl.ArrayOf(dsl.String), bounds)
+		dsl.Field(2, "labels", dsl.MapOf(dsl.String, dsl.String), bounds)
+		dsl.Field(3, "named", list)
+		dsl.Field(4, "required", dsl.ArrayOf(dsl.String), bounds)
+		dsl.Required("required")
+	})
+	arraySettings := dsl.Type("ArraySettings", func() {
+		dsl.Attribute("levels", dsl.ArrayOf(dsl.String), bounds)
+		dsl.Attribute("named", list)
+		dsl.Attribute("required", dsl.ArrayOf(dsl.String), bounds)
+		dsl.Required("required")
+	})
+	absent := dsl.Type("Absent", func() {
+		dsl.Attribute("missing", dsl.Boolean)
+		dsl.Required("missing")
+	})
+	choice := dsl.Type("Choice", dsl.OneOf(arraySettings, absent), func() {
+		dsl.Untagged()
+	})
+	dsl.Service("collections", func() {
+		dsl.Method("read", func() {
+			dsl.Result(choice)
+			dsl.HTTP(func() {
+				dsl.GET("/read")
+			})
+		})
+		dsl.Method("check", func() {
+			dsl.Payload(settings)
+			dsl.Result(settings)
+			dsl.HTTP(func() {
+				dsl.POST("/check")
+			})
+			dsl.GRPC(func() {})
+		})
+	})
+	dsl.Service("collectionsrpc", func() {
+		dsl.JSONRPC(func() {
+			dsl.POST("/rpc")
+		})
+		dsl.Method("check", func() {
+			dsl.Payload(settings)
+			dsl.Result(settings)
+			dsl.JSONRPC(func() {})
+		})
+	})
+}
+
+const optionalCollectionValidationHarness = `package collections_test
+
+import (
+	"context"
+	"encoding/json/v2"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	loomhttp "github.com/CaliLuke/loom/http"
+	svc "example.com/collections/gen/collections"
+	rpc "example.com/collections/gen/collectionsrpc"
+	httpserver "example.com/collections/gen/http/collections/server"
+	rpcserver "example.com/collections/gen/jsonrpc/collectionsrpc/server"
+)
+
+func TestCollectionPresence(t *testing.T) {
+	for _, tc := range []struct {
+		name, body string
+		valid, arraysValid bool
+	}{
+		{"omitted", "{\"required\":[\"a\",\"b\"]}", true, true},
+		{"supplied", "{\"required\":[\"a\",\"b\"],\"levels\":[\"a\",\"b\"],\"labels\":{\"a\":\"a\",\"b\":\"b\"},\"named\":[\"a\",\"b\"]}", true, true},
+		{"empty array", "{\"required\":[\"a\",\"b\"],\"levels\":[]}", false, false},
+		{"short array", "{\"required\":[\"a\",\"b\"],\"levels\":[\"a\"]}", false, false},
+		{"long array", "{\"required\":[\"a\",\"b\"],\"levels\":[\"a\",\"b\",\"c\",\"d\",\"e\"]}", false, false},
+		{"empty map", "{\"required\":[\"a\",\"b\"],\"labels\":{}}", false, true},
+		{"short map", "{\"required\":[\"a\",\"b\"],\"labels\":{\"a\":\"a\"}}", false, true},
+		{"long map", "{\"required\":[\"a\",\"b\"],\"labels\":{\"a\":\"a\",\"b\":\"b\",\"c\":\"c\",\"d\":\"d\",\"e\":\"e\"}}", false, true},
+		{"empty named", "{\"required\":[\"a\",\"b\"],\"named\":[]}", false, false},
+		{"missing required", "{}", false, false},
+		{"empty required", "{\"required\":[]}", false, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var value svc.Settings
+			require.NoError(t, json.Unmarshal([]byte(tc.body), &value))
+			err := svc.ValidateArraySettings(&svc.ArraySettings{Levels: value.Levels, Named: value.Named, Required: value.Required})
+			require.Equal(t, tc.arraysValid, err == nil, "%v", err)
+			for _, protocol := range []string{"http", "rpc"} {
+				t.Run(protocol, func(t *testing.T) {
+					calls := 0
+					endpoint := func(_ context.Context, payload any) (any, error) {
+						calls++
+						encoded, err := json.Marshal(payload)
+						require.NoError(t, err)
+						var got svc.Settings
+						require.NoError(t, json.Unmarshal(encoded, &got))
+						require.Equal(t, value, got)
+						return payload, nil
+					}
+					mux := loomhttp.NewMuxer()
+					path, body := "/check", tc.body
+					if protocol == "http" {
+						httpserver.Mount(mux, httpserver.New(&svc.Endpoints{Check: endpoint}, mux, loomhttp.RequestDecoder, loomhttp.ResponseEncoder, nil, nil))
+					} else {
+						rpcserver.Mount(mux, rpcserver.New(&rpc.Endpoints{Check: endpoint}, mux, loomhttp.RequestDecoder, loomhttp.ResponseEncoder, nil))
+						path = "/rpc"
+						body = "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"check\",\"params\":" + tc.body + "}"
+					}
+					recorder := httptest.NewRecorder()
+					request := httptest.NewRequest("POST", path, strings.NewReader(body))
+					request.Header.Set("Content-Type", "application/json")
+					mux.ServeHTTP(recorder, request)
+					if tc.valid {
+						require.Equal(t, 1, calls, recorder.Body.String())
+						require.Equal(t, 200, recorder.Code, recorder.Body.String())
+					} else {
+						require.Zero(t, calls)
+						if protocol == "http" {
+							require.Equal(t, 400, recorder.Code, recorder.Body.String())
+						} else {
+							require.Contains(t, recorder.Body.String(), "\"error\":")
+						}
+					}
+				})
+			}
+		})
+	}
+}
+`

codegen/testdata/golden/validation_array-pointer.go.golden

@@ -5,8 +5,10 @@ func Validate() (err error) {
 	if len(target.RequiredArray) < 5 {
 		err = loom.MergeErrors(err, loom.InvalidLengthError("target.required_array", len(target.RequiredArray), 5, true))
 	}
-	if len(target.DefaultArray) > 3 {
-		err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_array", len(target.DefaultArray), 3, false))
+	if target.DefaultArray != nil {
+		if len(target.DefaultArray) > 3 {
+			err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_array", len(target.DefaultArray), 3, false))
+		}
 	}
 	for _, e := range target.Array {
 		if !(e == 0 || e == 1 || e == 1 || e == 2 || e == 3 || e == 5) {

codegen/testdata/golden/validation_array-required.go.golden

@@ -5,8 +5,10 @@ func Validate() (err error) {
 	if len(target.RequiredArray) < 5 {
 		err = loom.MergeErrors(err, loom.InvalidLengthError("target.required_array", len(target.RequiredArray), 5, true))
 	}
-	if len(target.DefaultArray) > 3 {
-		err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_array", len(target.DefaultArray), 3, false))
+	if target.DefaultArray != nil {
+		if len(target.DefaultArray) > 3 {
+			err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_array", len(target.DefaultArray), 3, false))
+		}
 	}
 	for _, e := range target.Array {
 		if !(e == 0 || e == 1 || e == 1 || e == 2 || e == 3 || e == 5) {

codegen/testdata/golden/validation_array-use-default.go.golden

@@ -5,8 +5,10 @@ func Validate() (err error) {
 	if len(target.RequiredArray) < 5 {
 		err = loom.MergeErrors(err, loom.InvalidLengthError("target.required_array", len(target.RequiredArray), 5, true))
 	}
-	if len(target.DefaultArray) > 3 {
-		err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_array", len(target.DefaultArray), 3, false))
+	if target.DefaultArray != nil {
+		if len(target.DefaultArray) > 3 {
+			err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_array", len(target.DefaultArray), 3, false))
+		}
 	}
 	for _, e := range target.Array {
 		if !(e == 0 || e == 1 || e == 1 || e == 2 || e == 3 || e == 5) {

codegen/testdata/golden/validation_map-pointer.go.golden

@@ -7,8 +7,10 @@ func Validate() (err error) {
 	if len(target.RequiredMap) < 5 {
 		err = loom.MergeErrors(err, loom.InvalidLengthError("target.required_map", len(target.RequiredMap), 5, true))
 	}
-	if len(target.DefaultMap) > 3 {
-		err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_map", len(target.DefaultMap), 3, false))
+	if target.DefaultMap != nil {
+		if len(target.DefaultMap) > 3 {
+			err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_map", len(target.DefaultMap), 3, false))
+		}
 	}
 	for k, v := range target.Map {
 		err = loom.MergeErrors(err, loom.ValidatePatternCompiled("target.map.key", k, loomPatternTest0))

codegen/testdata/golden/validation_map-required.go.golden

@@ -7,8 +7,10 @@ func Validate() (err error) {
 	if len(target.RequiredMap) < 5 {
 		err = loom.MergeErrors(err, loom.InvalidLengthError("target.required_map", len(target.RequiredMap), 5, true))
 	}
-	if len(target.DefaultMap) > 3 {
-		err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_map", len(target.DefaultMap), 3, false))
+	if target.DefaultMap != nil {
+		if len(target.DefaultMap) > 3 {
+			err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_map", len(target.DefaultMap), 3, false))
+		}
 	}
 	for k, v := range target.Map {
 		err = loom.MergeErrors(err, loom.ValidatePatternCompiled("target.map.key", k, loomPatternTest0))

codegen/testdata/golden/validation_map-use-default.go.golden

@@ -7,8 +7,10 @@ func Validate() (err error) {
 	if len(target.RequiredMap) < 5 {
 		err = loom.MergeErrors(err, loom.InvalidLengthError("target.required_map", len(target.RequiredMap), 5, true))
 	}
-	if len(target.DefaultMap) > 3 {
-		err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_map", len(target.DefaultMap), 3, false))
+	if target.DefaultMap != nil {
+		if len(target.DefaultMap) > 3 {
+			err = loom.MergeErrors(err, loom.InvalidLengthError("target.default_map", len(target.DefaultMap), 3, false))
+		}
 	}
 	for k, v := range target.Map {
 		err = loom.MergeErrors(err, loom.ValidatePatternCompiled("target.map.key", k, loomPatternTest0))

codegen/tla/validation_carrier/CollectionPresence.tla

@@ -0,0 +1,26 @@
+--------------------- MODULE CollectionPresence ---------------------
+EXTENDS Naturals
+CONSTANT Mode
+VARIABLES required, supplied, size, nilIsEmpty, phase, valid
+vars == <<required, supplied, size, nilIsEmpty, phase, valid>>
+Init == /\ required \in BOOLEAN
+        /\ supplied \in BOOLEAN
+        /\ size \in 0..5
+        /\ nilIsEmpty \in BOOLEAN
+        /\ (nilIsEmpty => supplied)
+        /\ (~supplied => size = 0)
+        /\ phase = "input"
+        /\ valid = FALSE
+WithinBounds == size >= 2 /\ size <= 4
+Validate == /\ phase = "input"
+            /\ valid' = CASE Mode = "legacy" -> WithinBounds
+                         [] Mode = "native-only" ->
+                              (~required /\ (~supplied \/ (nilIsEmpty /\ size = 0))) \/ WithinBounds
+                         [] OTHER -> (~required /\ ~supplied) \/ WithinBounds
+            /\ phase' = "validated"
+            /\ UNCHANGED <<required, supplied, size, nilIsEmpty>>
+Next == Validate
+Spec == Init /\ [][Next]_vars
+Omission == phase = "validated" /\ ~required /\ ~supplied => valid
+Bounds == phase = "validated" /\ (required \/ supplied) => (valid = WithinBounds)
+=====================================================================

codegen/tla/validation_carrier/README.md

@@ -46,6 +46,43 @@ coverage deliberately abstracts physical Go carrier agreement. Primitive aliases
 byte slices and named Nullable declarations remain independent regression controls.
 Do not commit checker states or raw output; remove task-owned output after review.
 
+## Optional collections (#1042)
+
+`CollectionPresence.tla` distinguishes absent optional collections from supplied
+empty collections. It enumerates lengths zero through five with bounds two and
+four. Required occurrences also represent collection elements and extracted
+Optional/Nullable values, whose presence has already been established. It also
+models protobuf's nil-as-empty storage: a present collection wrapper supplies
+its collection even if decoding produces a nil slice/map. The model abstracts
+Go nil detection and does not model JSON decoding, defaults or naming.
+
+Run with the pinned tool above, substituting `CollectionPresence.tla`,
+`collection-legacy.cfg`, `collection-native-only.cfg` or `collection-checked.cfg`,
+and separate temporary state directories. Legacy fails `Omission` after 27
+distinct states: native absence is treated as a supplied empty collection.
+The first repair (`native-only`) fails `Bounds` after 29 states because protobuf
+nil contents bypass minimum length. Checked passes all 52 states, preserving
+bounds for supplied and required values. These runs used Go source base
+`05e01174e` and the candidate's explicit collection-storage context.
+
+`recurseValidationCode` owns the optional native array/map guard.
+`AttributeContext.CollectionNilIsEmpty`, set by `protoBufTypeContext` and retained
+by `Dup`, prevents that guard where native storage does not carry absence.
+Named calls retain their existing occurrence guards; required roots, collection
+elements and unwrapped presence values continue to validate concrete values.
+`TestOptionalCollectionValidationPresence` checks the emitted root guard and
+both bounds; `TestOptionalCollectionValidationGenerated` compiles service,
+HTTP, JSON-RPC and gRPC output, then exercises service validation and mounted
+HTTP/JSON-RPC omission, empty, valid, out-of-bound and required-field cases.
+`TestProtoCollectionPredicatesDoNotInferAbsence` checks the protobuf context;
+existing `TestCollectionWrapperGeneratedContracts` and
+`TestGeneratedStreamingRootValidation` reproduce the first repair's regression
+and verify corrected generated validation and mounted protobuf exchanges.
+The HTTP decoder goldens also retain the guard for optional array headers and
+query arrays/maps, including named collection aliases. `TestClientDecode` and
+`TestDecode` compare those emitted sections; element predicates remain outside
+the collection's nil guard and continue to validate supplied entries.
+
 ## Alias spine extraction (#892)
 
 `AliasPresence.tla` refines the root-versus-child distinction for inherited

codegen/tla/validation_carrier/collection-checked.cfg

@@ -0,0 +1,4 @@
+CONSTANT Mode = "checked"
+SPECIFICATION Spec
+INVARIANT Omission Bounds
+CHECK_DEADLOCK FALSE

codegen/tla/validation_carrier/collection-legacy.cfg

@@ -0,0 +1,4 @@
+CONSTANT Mode = "legacy"
+SPECIFICATION Spec
+INVARIANT Omission Bounds
+CHECK_DEADLOCK FALSE

codegen/tla/validation_carrier/collection-native-only.cfg

@@ -0,0 +1,4 @@
+CONSTANT Mode = "native-only"
+SPECIFICATION Spec
+INVARIANT Omission Bounds
+CHECK_DEADLOCK FALSE

codegen/transformer.go

@@ -60,6 +60,10 @@ type (
 		// map values use loom.Optional in a generated JSON decoding type so null
 		// is rejected by decoding before conversion to the service type.
 		CollectionElementPresence bool
+		// CollectionNilIsEmpty indicates that native collection storage cannot
+		// distinguish absence from empty, as with protobuf repeated/map fields.
+		// Validation must apply collection predicates even when that storage is nil.
+		CollectionNilIsEmpty bool
 		// JSONPresenceTypes records named types whose physical object fields use
 		// loom.Optional even when the root conversion itself uses native presence.
 		// It is keyed by the hash of the user type, which names its Go type.
@@ -375,6 +379,7 @@ func (a *AttributeContext) Dup() *AttributeContext {
 		SamePackageConversion:     a.SamePackageConversion,
 		JSONPresence:              a.JSONPresence,
 		CollectionElementPresence: a.CollectionElementPresence,
+		CollectionNilIsEmpty:      a.CollectionNilIsEmpty,
 		JSONPresenceTypes:         a.JSONPresenceTypes,
 		PresencePointerTypes:      a.PresencePointerTypes,
 		PresenceUseDefaultTypes:   a.PresenceUseDefaultTypes,