Translate the new-format custom constraints, with relaxation and expansion-limit constraints

CLAUDE.md

@@ -72,8 +72,11 @@ Conventions:
 
 - Use a plain CSV-like table format for DataFrame inputs and outputs — no need to
   wrap in runnable `csv_str_to_df` calls, since this is illustrative, not a doctest.
-- Abbreviate long column names when they would otherwise overflow the line; point at
-  the relevant constants for the real names.
+- Always lay DataFrame examples out as full tables — a header row plus example rows,
+  each table in its own block — even when a single row would do. Never condense a
+  table into `column=value` prose. Abbreviated column sets, row sets and column
+  names are fine when the full table would otherwise be large; point at the
+  relevant constants for the real names.
 - Cover representative edge cases in the same example, with trailing `# comment`
   notes on the rows that demonstrate each case.
 - For trivial utility functions, one-line input → output cases are enough.

src/ispypsa/templater/custom_constraints_from_plexos.py

@@ -156,9 +156,12 @@
 date_from is retained so time-varying coefficients carry through to the
 output.
 
-TODO: switch the IASR ID lookup to a templated generator-summary table once
-one exists -- matching against the raw table currently also matches DER/CER
-rows that ISPyPSA may not template.
+TODO: switch the IASR ID lookup to the templated generator and storage
+tables once the existing-unit ones exist -- matching against the raw summary
+tables currently also matches DER/CER rows that ISPyPSA may not template.
+The custom_constraints_lhs schema's variable_names_resolve_by_term_type rule
+declares that end state; note the new-entrant templater renames units to
+"<geo_id> <technology>", which the lookup here will need to follow.
 
 TODO: emitted timeslices are region-prefixed (qld_peak_demand etc.) while
 the rest of the templater still emits the bare canonical names

src/ispypsa/translator/mappings.py

@@ -152,14 +152,20 @@
     "generator_capacity": "Generator",
     "generator_output": "Generator",
     "load_consumption": "Load",
+    # "load" is the new-format vocabulary for "load_consumption" (PLEXOS Node
+    # Load Coefficient terms). A load term is a data term — the demand (p_set)
+    # at the term's demand node, scaled by the coefficient — rather than a
+    # term on an optimisation variable.
+    "load": "Load",
     "storage_output": "Storage",
 }
 
 _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE = {
     "link_flow": "p",
     "generator_capacity": "p_nom",
     "generator_output": "p",
-    "load_consumption": "p",
+    "load_consumption": "p_set",
+    "load": "p_set",
     "storage_output": "p",
 }
 

src/ispypsa/translator/network.py

@@ -398,7 +398,7 @@ def _enabled_expansion_element_ids(
     ``transmission_expansion`` gates flow paths between (sub)regions;
     ``rez_transmission_expansion`` gates REZ connection paths. The result is the
     enabled expansion_id set the options and costs tables are filtered to (see
-    _keep_rows_for_enabled_elements) and then resolved against, so an option or
+    _keep_rows_for_expansion_ids) and then resolved against, so an option or
     cost for a disabled or non-modelled element drops out before resolution.
 
     I/O Example:
@@ -462,7 +462,7 @@ def _resolve_expansion_options(
     # not dropped, so they are set aside before wildcard resolution (which would
     # otherwise raise on them).
     options = options[options["expansion_type"] != "constraint_relaxation"]
-    options = _keep_rows_for_enabled_elements(options, enabled_ids)
+    options = _keep_rows_for_expansion_ids(options, enabled_ids)
     allowed_values = {
         "expansion_id": enabled_ids,
         "expansion_type": ["forward", "reverse"],
@@ -477,22 +477,23 @@ def _resolve_expansion_options(
     return options
 
 
-def _keep_rows_for_enabled_elements(
-    table: pd.DataFrame, enabled_ids: list[str]
+def _keep_rows_for_expansion_ids(
+    table: pd.DataFrame, expansion_ids: list[str]
 ) -> pd.DataFrame:
-    """Keeps rows whose expansion_id is an enabled element or blank (a wildcard).
+    """Keeps rows whose expansion_id is one of the given ids or blank (a wildcard).
 
-    Selecting the enabled elements is config-driven, so it happens before
-    wildcard resolution rather than inside it. Rows for disabled or non-modelled
-    elements drop out here, as do rows for constraint groups (their expansion_ids
-    are constraint_ids, routed to ispypsa.translator.constraints instead).
+    Which ids to keep is a config-driven selection made by the caller — the
+    enabled paths here, the constraints in the model in
+    ispypsa.translator.constraints — so it happens before wildcard resolution
+    rather than inside it. Rows for any other id drop out; the blank rows are
+    kept because they are wildcards that resolve against the given ids.
 
     I/O Example:
-        table:                                enabled_ids = ["CQ-NQ"]
+        table:                                expansion_ids = ["CQ-NQ"]
             expansion_id  year  cost
             CQ-NQ         2026  1000000
-            Q1-NQ         2026  500000    # disabled element: dropped
-            SWQLD1        2026  400000    # constraint group: dropped
+            Q1-NQ         2026  500000    # not in expansion_ids: dropped
+            SWQLD1        2026  400000    # not in expansion_ids: dropped
                           2026  900000    # blank: a wildcard, kept
 
         returns:
@@ -501,7 +502,7 @@ def _keep_rows_for_enabled_elements(
                           2026  900000
     """
     ids = table["expansion_id"]
-    keep = ids.isna() | ids.isin(enabled_ids)
+    keep = ids.isna() | ids.isin(expansion_ids)
     return table[keep]
 
 
@@ -591,19 +592,20 @@ def _prepare_expansion_costs(
     wacc: float,
     asset_lifetime: int,
 ) -> pd.DataFrame:
-    """Resolves the expansion-cost wildcards to the enabled elements and
+    """Resolves the expansion-cost wildcards to the given expansion ids and
     investment periods, then annuitises them.
 
     Blank expansion_id or year cells are wildcards (see the
     network_transmission_path_expansion_costs schema): an empty expansion_id is a
     table-wide default cost, an empty year a static cost across the investment
-    periods. Costs for disabled elements, constraint groups (routed to
-    ispypsa.translator.constraints) and years outside the investment periods are
-    all designed selection, filtered out first. _resolve_wildcards
-    then expands the blanks against the enabled elements and the investment
-    periods. A blank cost then resolves to free — the nan_fill the schema
-    declares for the cost column. Year values are labels matched against the
-    config's investment periods as ints — no financial vs calendar year
+    periods. The caller decides which ids the costs are for — the enabled paths
+    here, the constraints in the model in ispypsa.translator.constraints — so
+    rows for any other id, and rows for years that are not investment periods,
+    are dropped before resolution: they are selection, not bad data.
+    _resolve_wildcards then expands the blanks against the given ids and the
+    investment periods. A blank cost then resolves to free — the nan_fill the
+    schema declares for the cost column. Year values are labels matched against
+    the config's investment periods as ints — no financial vs calendar year
     interpretation happens here; the config's year_type decides what span of
     time the labels denote, so the table just has to label years the same way
     the config does (templated tables carry financial-year ending years).
@@ -622,7 +624,7 @@ def _prepare_expansion_costs(
             CQ-NQ         2026  annuitise(1000)   # the static row fills 2026
             CQ-NQ         2028  annuitise(1200)   # the 2028 override beats the static row
     """
-    costs = _keep_rows_for_enabled_elements(expansion_costs, enabled_ids)
+    costs = _keep_rows_for_expansion_ids(expansion_costs, enabled_ids)
     costs = _keep_rows_for_investment_period_years(costs, investment_periods)
     allowed_values = {"expansion_id": enabled_ids, "year": investment_periods}
     costs = _resolve_wildcards(costs, allowed_values, ["cost"])

src/ispypsa/validation/schemas/custom_constraints_lhs.yaml

@@ -11,14 +11,30 @@ description: >
       templater.
 
   Source notes:
-  variable_name uses IASR IDs for generators and batteries and path_ids for
-  links. Generator and battery terms may reference units that are not in the
-  model (e.g. before generator templating lands, or units outside a filtered
-  region); such terms are skipped with a log line when constraints are
-  applied.
+  variable_name uses IASR IDs for generators and batteries, path_ids for links
+  and sub-region codes for loads. The variable_names_resolve_by_term_type rule
+  ties each to its component table, so the translator and pypsa_build take the
+  names as given rather than checking them against the model.
 
   If absent:
   Constraints have no terms, so no custom constraints bind.
+custom_validation:
+  - name: lhs_and_rhs_name_the_same_constraints
+    description: >
+      Every constraint_id with a row here must have at least one row in
+      custom_constraints_rhs, and every constraint_id in custom_constraints_rhs
+      must have at least one row here. A constraint with terms but no limit
+      value, or a limit value but no terms, cannot be applied. This does not
+      require every custom_constraints row to have terms: a catalogued
+      constraint with neither LHS nor RHS rows simply never binds.
+  - name: variable_names_resolve_by_term_type
+    description: >
+      Each row's variable_name must exist in the table its term_type refers to:
+      link_flow -> network_transmission_paths.path_id; generator_output and
+      generator_capacity -> generators_existing_planned.name or
+      generators_new_entrant.name; storage_output -> storage_existing_planned.name
+      or storage_new_entrant.name; load -> network_geography.geo_id. A term
+      naming a component that is not in those tables cannot be applied.
 columns:
   constraint_id:
     type: string
@@ -39,9 +55,19 @@ columns:
     type: string
     required: true
     description: >
-      Name of the component the term's variable belongs to: an IASR ID for
-      generator_output and storage_output terms, a path_id for link_flow
-      terms.
+      Name of the model element the term's variable belongs to: an IASR ID for
+      generator_output, generator_capacity and storage_output terms, a path_id
+      for link_flow terms, a sub-region geo_id for load terms (see
+      custom_validation for the table each resolves against).
+
+      Names refer to whole elements, never to the per-build-year components
+      the translator builds from them, and one term covers every component of
+      its element. A new entrant generator or storage unit is referred to by
+      its name in generators_new_entrant / storage_new_entrant (its location
+      and technology, e.g. "N2 Solar"), and the term applies to the unit in
+      every build year. A path is referred to by its path_id (e.g. "NSW-QLD"),
+      and the term applies to the path's existing link and each of its
+      expansion links.
   coefficient:
     type: float
     required: true

src/ispypsa/validation/schemas/network_expansion_options.yaml

@@ -48,6 +48,14 @@ custom_validation:
       direction and stands alone. This keeps an element's resolved forward and
       reverse tied to the same option, so the single cost keyed on expansion_id
       applies to a coherent pair.
+  - name: relaxation_options_target_inequality_constraints
+    description: >
+      A constraint_relaxation row must not resolve to a constraint whose
+      direction in custom_constraints is "=", whether keyed on the constraint
+      directly or reached through a blank expansion_id or expansion_type
+      wildcard. Relaxation adds a single-signed slack term to the constraint's
+      LHS, which can loosen a "<=" or ">=" but can only move an equality one
+      way.
 columns:
   expansion_id:
     type: string