diff --git a/.codespellrc b/.codespellrc index 19cd4cf..0eec3a4 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] skip = .git,.conda,*.json,*.001,*.002,*.map,*.tif,*.tiff,*.png,*.pdf,*.lock,doc/source/generated,doc/source/_static,doc/_build,tests/fixtures/base -ignore-words-list = pervious,tese,hims,fo +ignore-words-list = pervious,tese,hims,fo,botton diff --git a/README.md b/README.md index f67651c..95eff74 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,9 @@ The model was developed based on classical concepts of hydrological processes an - Distributed monthly step model; - Hydrological process based on soil water balance in each pixel, and flow total calculated after composition of the resulting accumulated flow, according to Direction drainage network flow established by the digital elevation model (DEM); - Calculations for two zones: rootzone and saturated; -- Evapotranspiration and interception process based on vegetation index: Leaf Area Index (LAI), Photosynthetically Active Radiation Fraction (FPAR) and Normalized Difference Vegetation Index (NDVI); and -- Sub-pixel level coverage classification, represented by four fractions that represent percentage of total pixel area covered exclusively by: area vegetated, bare soil area, water area and impervious area. +- Evapotranspiration and interception process based on vegetation index: Leaf Area Index (LAI), Photosynthetically Active Radiation Fraction (FPAR) and Normalized Difference Vegetation Index (NDVI); +- Sub-pixel level coverage classification, represented by four fractions that represent percentage of total pixel area covered exclusively by: area vegetated, bare soil area, water area and impervious area; and +- Optional groundwater coupling with MODFLOW-2005 through the PCRaster MODFLOW extension: the recharge feeds a layered aquifer model and the aquifer-to-river leakage becomes the baseflow. ## Getting Started diff --git a/doc/source/calibration.rst b/doc/source/calibration.rst index e97978d..6f4dfe3 100644 --- a/doc/source/calibration.rst +++ b/doc/source/calibration.rst @@ -40,7 +40,9 @@ must be in the same quantity. Each evaluation runs the whole simulation period of the configuration. The calibration never changes the period, the inputs or the aggregation: of what -the results depend on, it changes the nine parameters and nothing else. +the results depend on, it changes the nine parameters and nothing else, apart +from the parameters of the groundwater model a calibration of a coupled +configuration names, see `MODFLOW parameters`_. The objective function ---------------------- @@ -329,6 +331,101 @@ that starting point which a narrowed bound excludes is moved onto the bound it crosses, and the move is logged: the run starts from the admissible point closest to the configuration rather than ending before its first evaluation. +MODFLOW parameters +------------------ + +A configuration that enables the :doc:`MODFLOW coupling ` is +calibrated like any other: no option is needed, and every evaluation runs the +coupled model. Its section also offers parameters of the groundwater model to +the search. They are named after where they live in the section, with the +layers numbered from the top down, as in the configuration: + +.. list-table:: MODFLOW parameters + :header-rows: 1 + :widths: 34 36 12 18 + + * - Name + - Parameter + - Unit + - Values it may take + * - ``modflow.layers..specific_yield`` + - Specific yield of layer ``n`` + - – + - :math:`(0, 1]` + * - ``modflow.layers..specific_storage`` + - Confined storage coefficient of layer ``n`` + - – + - :math:`(0, \infty)` + * - ``modflow.layers..kh.`` + - Horizontal conductivity of one class of the lookup table of layer ``n`` + - m/day + - :math:`(0, \infty)` + * - ``modflow.river..conductance`` + - Conductance of every river cell of river entry ``i``, counted from 1 + - :raw-html:`m2day-1` + - :math:`(0, \infty)` + +A name exists only where the configuration gives a number that the run reads; +a raster is not a number to search: + +- the storage of a layer, when the section gives it as a number, the run is + transient and the layer type reads it: ``LAYCON`` 0 reads the specific + storage, ``LAYCON`` 1 the specific yield, ``LAYCON`` 2 and 3 both; +- the conductivity of a class, when the horizontal conductivity of the layer + is a class ``map`` with a lookup ``table``: one name per class of the + table, spelt as a number in its shortest form (``01`` and ``1.0`` are class + ``1``). Only the row PCRaster reads for a class is named: an interval row + is not, nor a row whose class an earlier row, numeric or interval, already + matches; +- the conductance of a river entry, when it is a number (with its ``mask``). + +A MODFLOW parameter is **never searched by default**. It joins the search only +when ``--bound`` names it, and the bound is mandatory, since the application +settings have no range for it: it must be finite, its minimum below its +maximum, and both inside the values the parameter may take. ``--fix`` pins +one at a value inside the same values instead, and naming a parameter in both +is refused. A name that is not a MODFLOW parameter of the configuration is +refused with the list of the ones it offers, and a MODFLOW name given for a +configuration that does not enable MODFLOW is refused as well; all of that +before the search starts. + +.. code-block:: console + + $ rubem calibrate -c project-config.json --observed observed.csv -o calibration \ + --bound modflow.layers.1.specific_yield=0.05:0.3 \ + --bound modflow.layers.1.kh.2=0.05:0.5 --fix alpha_gw=0.5 + +The searched MODFLOW parameters follow the eight parameters of the model in the +decision vector, so each of them is one more dimension in +`The evaluation budget`_, and a search may leave every parameter of the model +fixed and search MODFLOW parameters only. The starting point takes their +values from the configuration, moved onto the bound like the others. + +Every candidate carries its MODFLOW values into the configuration of its +evaluation: the numbers replace the ones of the section, and the classes of a +lookup table are written, with the other rows of the table unchanged, into +:file:`kh_layer.tbl` in the temporary directory of the evaluation, which the +layer then reads; the configured table is never modified. The parameters that +were searched or fixed appear in :file:`evaluations.csv`, in +:file:`result.json` and in the calibrated configuration, see +`Artifacts of a run`_. + +.. note:: + + In a coupled run the baseflow comes from MODFLOW, and the recession + coefficient :math:`\alpha_{GW}` has no effect on the simulation (see + :ref:`groundwater:The saturated zone`). Fix it, as in the example above, + so that the search does not spend a dimension on it. + +.. warning:: + + With ``dis.nstp`` above 1, a candidate whose MODFLOW solver fails before + the last time step of a stress period ends its worker process, and with it + the calibration, instead of being recorded as a failed evaluation; see + :ref:`groundwater:Limitations`. The command warns about it before the + search starts. With the default ``dis.nstp`` of 1 such a candidate is a + failed evaluation like any other. + The search ---------- @@ -416,8 +513,9 @@ The number of population members is not ``--popsize``: it is \[members = \max\left(5,\ popsize \times N\right)\] with :math:`N` the number of free parameters — eight, minus one for every -``--fix``. With ``--init sobol``, the default, that count is then rounded up to -the next power of two, because a Sobol' sequence is balanced only over a +``--fix``, plus one for every bounded parameter of `MODFLOW parameters`_. With +``--init sobol``, the default, that count is then rounded up to the next power +of two, because a Sobol' sequence is balanced only over a power-of-two sample; the other three initializations use it as it is. Every generation evaluates one candidate per member, and the initial population is one more round of evaluations on top of ``--maxiter`` generations, so @@ -624,8 +722,9 @@ with; :file:`evaluations.csv` is comma-separated. One row per evaluation, with the columns ``id``, ``pid``, ``started_at``, ``alpha``, ``beta``, ``w_1``, ``w_2``, ``w_3``, ``rcd``, ``f``, ``alpha_gw``, ``x``, ``nse``, ``objective``, ``elapsed_seconds`` and - ``error``. ``id`` identifies the evaluation, ``pid`` the process that made - it and ``started_at`` the wall-clock moment it began, in UTC; ``nse`` is + ``error``; the `MODFLOW parameters`_ a run searched or fixed follow ``x``. + ``id`` identifies the evaluation, ``pid`` the process that made it and + ``started_at`` the wall-clock moment it began, in UTC; ``nse`` is empty for a candidate that produced no efficiency; ``error`` is empty for a successful evaluation, ``inadmissible`` for a candidate rejected without a run, and the type and message of the exception for a run that failed. The @@ -659,10 +758,11 @@ with; :file:`evaluations.csv` is comma-separated. :file:`result.json` The summary: ``best_parameters`` (the nine parameters, the derived ``w_3`` - included), ``best_nse``, ``best_objective``, ``nfev`` and ``nit`` (the - evaluations and generations the search spent), ``success`` and ``message`` - from the optimizer, ``population_size``, an ``artifacts`` object naming - every file above, and a ``settings`` object with ``variable``, + included, then the MODFLOW parameters searched or fixed), ``best_nse``, + ``best_objective``, ``nfev`` and ``nit`` (the evaluations and generations + the search spent), ``success`` and ``message`` from the optimizer, + ``population_size``, an ``artifacts`` object naming every file above, and a + ``settings`` object with ``variable``, ``spinup_steps``, ``maxiter``, ``popsize``, ``seed``, ``workers``, ``temp_dir``, ``init``, ``strategy``, ``mutation``, ``recombination``, ``polish``, ``bounds``, ``fixed`` and ``stations``. The workers, the @@ -676,7 +776,11 @@ with; :file:`evaluations.csv` is comma-separated. in place, written in the format the input was written in, legacy or format 1.0. Its paths are the ones the loader resolved, which are absolute even when the input file wrote them relative to its own directory; moving - the file to another machine therefore means fixing the paths. + the file to another machine therefore means fixing the paths. The MODFLOW + parameters of the best candidate replace the ones of its section, and a + conductivity table with a calibrated class is written next to it as + :file:`-calibrated-kh.tbl`, which the calibrated configuration + points at. Next to them, the :file:`evaluations/` directory holds one JSON record per evaluation, written by the worker that made it. A record carries what the CSV @@ -722,8 +826,9 @@ candidate and exits. The generation is spread over as many such workers as ``--workers`` allows, which is also what makes the calibration parallel at all. Each worker builds the configuration of its candidate from the document of the -calibrated configuration: the nine parameters become the candidate's, the -output directory becomes a temporary directory of its own under ``--temp-dir``, +calibrated configuration: the nine parameters become the candidate's (and so do +the `MODFLOW parameters`_ of the run), the output directory becomes a temporary +directory of its own under ``--temp-dir``, every raster series is disabled (an evaluation reads no raster of a previous run, and writing them would dominate the cost of the run) and the time series are reduced to the calibrated variable, as CSV. The aggregation the user @@ -780,7 +885,9 @@ A worker that disappears before it finishes its evaluation ends the calibration, and the most common reason is the system killing it for lack of memory: every worker holds the rasters of one model run. The message says so, advises a lower ``--workers`` and names the table of the evaluations made up to -that point, which is written on the way out. +that point, which is written on the way out. With MODFLOW enabled it also names +the other cause, a solver failure before the last time step of a stress period +when ``dis.nstp`` is above 1 (see `MODFLOW parameters`_). Definitions worth knowing ------------------------- diff --git a/doc/source/changelog.rst b/doc/source/changelog.rst index 58d2a07..abc1357 100644 --- a/doc/source/changelog.rst +++ b/doc/source/changelog.rst @@ -13,6 +13,35 @@ The format follows `Keep a Changelog `__. Added ````` +- Added the optional coupling with MODFLOW-2005 through the PCRaster MODFLOW + extension (``MODFLOW`` section, format 1.0 ``modflow``; ignored when absent + or disabled, and reported as ignored when present but disabled). The + recharge of each step feeds the RCH package, one step being one stress + period of the days of its month, and the leakage from the aquifer to the RIV + cells becomes ``bfw``, in mm, while the saturated-zone reservoir is no + longer updated; GHB, DRN and BCF wetting shape the heads. Layers are listed + from the top down and numbered like MODFLOW (at most 9), with explicit layer + lists for every package; the configuration refuses ``LAYCON`` 1 below the + top layer, wetting on layers that are not ``LAYCON`` 1 or 3 and a transient + layer without the storage its type reads, and the validation checks the + geometry, the heads against the bottoms and the no-data markers, the + conductivity tables, the + package cells, reports river cells whose bed lies outside their layer, and + blocks, even with ``-s``, a missing input file or a missing MODFLOW runtime + (the extension and ``mf2005``, both in the conda-forge ``pcraster`` + package). MODFLOW runs in ``/modflow``, removed after the last step; + heads, river exchange, storage, drain flow and an experimental root-depth + coupling with the water table are written as raster series selected in the + section. A stress period has one time step unless ``dis.nstp`` says + otherwise, so that a solver failure raises an error instead of ending the + process. ``rubem calibrate`` runs the coupled model whenever the + configuration enables it, warns when ``dis.nstp`` is above 1, and searches + or fixes the MODFLOW parameters + named in ``--bound`` and ``--fix`` (``modflow.layers..specific_yield``, + ``modflow.layers..specific_storage``, ``modflow.layers..kh.``, + ``modflow.river..conductance``). The new Groundwater Coupling page + documents the exchange, the keys, the rules, the outputs and the + limitations (`#356 `__). - Added ``rubem calibrate``: a differential evolution (SciPy) over the free calibration parameters, with ``w3`` derived from the other two weights, minimizing ``1000 (100 (1 - NSE))^2`` on the station series of one output diff --git a/doc/source/groundwater.rst b/doc/source/groundwater.rst new file mode 100644 index 0000000..6a79e1b --- /dev/null +++ b/doc/source/groundwater.rst @@ -0,0 +1,973 @@ +Groundwater Coupling with MODFLOW +================================= + +.. role:: raw-html(raw) + :format: html + +RUBEM represents the saturated zone of each cell as a lumped linear reservoir: +the recharge fills it and the baseflow :math:`BFW` drains it with the +recession coefficient :math:`\alpha_{GW}` (see +:ref:`overview:Calibration and Validation`). A basin whose regional aquifer, +river-aquifer exchange or lateral boundaries matter cannot be represented that +way. The optional MODFLOW section replaces that reservoir by a +three-dimensional groundwater flow model, MODFLOW-2005 [HARBAUGH2005]_, run +through the MODFLOW extension of PCRaster (``pcraster.initialise``). + +The coupling is **opt-in**. A configuration without the section, or with the +section and ``"enabled": false``, runs exactly as before: nothing of this page +applies to it, and RUBEM does not even load its groundwater module. A section +present but disabled is reported as ignored (a non-blocking problem); its keys +must still be keys of the section, but its rules and its files are not +checked. + +This page documents what the coupling exchanges, the configuration keys and +their units, the rules the configuration and the inputs must follow, what the +run writes, and the limitations of the coupling. The keys are also listed, +section by section, in the :ref:`user guide `. + +What the coupling exchanges +--------------------------- + +One step, one stress period +``````````````````````````` + +Every RUBEM time step is one MODFLOW stress period. RUBEM steps are months, so +the length of a period, :math:`\Delta t`, is the number of days of the month +of the step; in a transient run the period is divided into ``dis.nstp`` time +steps. The groundwater model is built once, when the simulation starts, and +kept for the whole run: the heads of a period are the initial heads of the +next one. + +Recharge +```````` + +The recharge RUBEM computes for the step (the ``rec`` output variable, in +millimetres per step) is given to the MODFLOW recharge package (RCH) as a flux +in metres per day: + +.. math:: + :label: groundwater-recharge + :nowrap: + + \[R_{MF} = \frac{REC}{1000 \cdot \Delta t}\] + +where: + +- :math:`R_{MF}` – recharge flux given to MODFLOW [m/day]; +- :math:`REC` – recharge of the step computed by RUBEM [mm]; +- :math:`\Delta t` – length of the stress period [days]. + +A cell without recharge (missing value) receives none. The recharge is applied +to the highest active cell of each column (RCH option ``NRCHOP = 3``). + +Baseflow +```````` + +The river package (RIV) exchanges water between the aquifer and the river +cells. The leakage that leaves the aquifer towards the river is the baseflow +of the step: it replaces the baseflow of the lumped reservoir and keeps the +output variable ``bfw`` and its unit, millimetres per step. + +.. math:: + :label: groundwater-baseflow + :nowrap: + + \[BFW = \frac{1000 \cdot \Delta t}{A} \sum_{k} \max\left(-Q_{RIV,k},\ 0\right)\] + +where: + +- :math:`BFW` – baseflow of the step [mm]; +- :math:`Q_{RIV,k}` – RIV leakage of the cell in layer :math:`k`, positive + into the aquifer [:raw-html:`m3day-1`]; the sum runs + over the layers that hold river cells; +- :math:`A` – area of a RUBEM cell, the square of ``GRID.grid`` + (``raster_info.grid_size`` in format 1.0) [:raw-html:`m2`]; +- :math:`\Delta t` – length of the stress period [days]. + +The baseflow enters the total runoff of the cell as it does without MODFLOW. +The leakage in the opposite direction, from the river into the aquifer, is +reported (``mfrv2aq``, see `Outputs of a coupled run`_) but is not taken from +any RUBEM flux. + +The saturated zone +`````````````````` + +While MODFLOW is coupled it holds the saturated zone: the lumped reservoir of +RUBEM is not updated, and the baseflow recession equation is not evaluated. +The parameters and initial conditions that only feed that reservoir, +``alpha_gw`` (``CALIBRATION``) and ``bfw_ini``, ``bfw_lim`` and ``S_sat_ini`` +(``INITIAL_SOIL_CONDITIONS``), remain mandatory keys of the configuration and +are still validated, but they have no effect on a coupled run. + +Boundaries that shape the heads +``````````````````````````````` + +General-head boundaries (GHB), drains (DRN) and the rewetting of dry cells +(BCF wetting) change the heads, and through them the river leakage, but none of +their flows is returned to RUBEM. + +Requirements +------------ + +The coupling needs two things that the conda-forge ``pcraster`` package +installs with PCRaster itself, so no other package is needed: + +- the PCRaster MODFLOW extension, the module ``pcraster._pcraster_modflow``; +- the MODFLOW-2005 executable ``mf2005``, which the extension starts. + +The extension looks for ``mf2005`` on ``PATH``. When the environment is used +without being activated it is not there, so RUBEM also looks next to the +interpreter, in ``/bin/mf2005`` (or ``/Library/bin/mf2005.exe`` +on Windows), and puts that directory at the front of ``PATH`` for the run and +its calibration workers, which it logs. + +When either of them is missing, the configuration of an enabled section +carries the blocking problem ``MODFLOW runtime is not available.``, with the +installation guidance. This check, like the existence of the MODFLOW input +files, runs even when the validation of the inputs is skipped with ``-s``: +without them the coupled run cannot start. + +Layers +------ + +The layers are listed **from the top down**, and numbered like MODFLOW: layer +``1`` is the top layer. That number is the one every key, message and output +name uses (``river.entries[].layers``, ``wetting.layers``, +``water_table.layer``, ``mfh1``, ``modflow.layers.1.specific_yield``). The +PCRaster MODFLOW extension numbers its layers the other way round, from the +bottom up; RUBEM converts the numbers itself, and the user never writes the +extension's numbers. + +The geometry is a model top and one bottom per layer; the top of a layer is +the model top for layer 1 and the bottom of the layer above for the others: + +.. code-block:: text + + ------------------------------ model top top + layer 1 (top layer) + ------------------------------ bottom of layer 1 layers[0].bottom + layer 2 + ------------------------------ bottom of layer 2 layers[1].bottom + layer 3 (base layer) + ------------------------------ bottom of layer 3 layers[2].bottom + +A section lists at most **9** layers: the name of a per-layer output is its +prefix followed by the layer number and the step (``mfh1`` then the step), so +layer 1 and layer 10 would give the same file names. + +On the columns that are active in at least one layer, the top and every bottom +must have a value and each surface must lie strictly below the one above it. +The columns that are inactive in every layer lie outside the groundwater +domain; MODFLOW still needs elevations there, so RUBEM gives them synthetic +layers one metre thick, in memory. The input files are never modified. + +The MODFLOW grid has one cell per RUBEM cell, and every MODFLOW cell is a +square whose side is ``GRID.grid`` metres: the row and column widths come from +the grid size of the configuration, whatever the units of the clone. On a +geographic clone this is the square-cell approximation RUBEM already makes for +its own fluxes, not a reprojection. + +The configuration section +------------------------- + +The section is ``modflow`` in configuration format 1.0 and ``MODFLOW`` in the +legacy format; the keys inside it are the same in both. Every file path is +anchored on the directory of the configuration file when it is relative, like +the other inputs. The section is strict in both formats: a key it does not +know is refused when the configuration is loaded, instead of being ignored. + +Lengths are in metres and times in days, the units of the coupling; they are +not configurable. + +The example below is a three-layer model with a river on the top layer and +general-head boundaries on the three layers (format 1.0): + +.. code-block:: json + + { + "modflow": { + "enabled": true, + "top": "input/modflow/top_model.map", + "layers": [ + { + "name": "upper", + "bottom": "input/modflow/botton3.map", + "initial_head": "input/modflow/head3.map", + "boundary": "input/modflow/bound.map", + "laytype": 1, + "horizontal_conductivity": { + "map": "input/modflow/kh_classes3.map", + "table": "input/modflow/kh3.tbl" + }, + "vertical_conductivity": "input/modflow/KY3.map", + "specific_yield": 0.15, + "specific_storage": 1e-6 + }, + { + "name": "middle", + "bottom": "input/modflow/botton2.map", + "initial_head": "input/modflow/head2.map", + "boundary": "input/modflow/bound.map", + "laytype": 2, + "horizontal_conductivity": { + "map": "input/modflow/kh_classes2.map", + "table": "input/modflow/kh2.tbl" + }, + "vertical_conductivity": "input/modflow/KY2.map", + "specific_yield": 0.15, + "specific_storage": 1e-6 + }, + { + "name": "lower", + "bottom": "input/modflow/botton.map", + "initial_head": "input/modflow/head1.map", + "boundary": "input/modflow/bound.map", + "laytype": 2, + "horizontal_conductivity": { + "map": "input/modflow/kh_classes1.map", + "table": "input/modflow/kh1.tbl" + }, + "vertical_conductivity": "input/modflow/KY1.map", + "specific_yield": 0.15, + "specific_storage": 1e-6 + } + ], + "dis": {"nstp": 5, "tsmult": 1.0, "steady_state": false}, + "solver": { + "mxiter": 2000, + "iter1": 20, + "npcond": 1, + "hclose": 5.0, + "rclose": 3.0, + "relax": 1.0, + "nbpol": 2, + "damp": 0.5 + }, + "wetting": { + "enabled": true, + "map": "input/modflow/wet.map", + "layers": [1], + "wetfct": 1.0, + "iwetit": 3, + "ihdwet": 0 + }, + "river": { + "enabled": true, + "entries": [ + { + "layers": [1], + "stage": "input/modflow/riv_stage.map", + "bottom": "input/modflow/riv_bot.map", + "conductance": 0.387, + "mask": "input/modflow/riv_cond.map" + } + ] + }, + "ghb": { + "enabled": true, + "entries": [ + { + "layers": [1, 2, 3], + "head": "input/modflow/ghb_head.map", + "conductance": "input/modflow/ghb_cond.map" + } + ] + }, + "drain": {"enabled": false, "entries": []}, + "coupling": { + "dynamic_root_depth": { + "enabled": false, + "minimum_depth_table": null, + "water_table": {"method": "highest_active_head", "layer": null} + } + }, + "output": { + "heads": true, + "river_leakage": true, + "storage": false, + "drain_flow": false, + "root_depth": false + } + } + } + +The tables below list every key. A *raster* is a path to a map on the grid of +the clone; where a key accepts a raster or a number, the number is written +without quotes (``0.15`` is a value, ``"0.15"`` is a file name) and stands for +that value in every cell. + +Section keys +```````````` + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``enabled`` + - boolean, default ``false`` + - – + - Whether the run is coupled to MODFLOW. The other keys are checked only + when it is ``true``. + * - ``top`` + - raster, required + - m + - Elevation of the model top, the top of layer 1. + * - ``layers`` + - list of 1 to 9 layers, required + - – + - The layers from the top down, see `Layer keys`_. + * - ``dis`` + - object + - – + - Time discretization, see `Time discretization`_. + * - ``solver`` + - object + - – + - PCG solver settings, see `Solver`_. + * - ``wetting`` + - object + - – + - Rewetting of dry cells, see `Wetting`_. + * - ``river``, ``ghb``, ``drain`` + - objects + - – + - The stress packages, see `Rivers, general-head boundaries and drains`_. + ``river.enabled`` must be ``true``: the river leakage is the baseflow. + * - ``coupling`` + - object + - – + - The root-depth coupling, see `The root-depth coupling`_. + * - ``output`` + - object + - – + - The diagnostics written as rasters, see `Outputs of a coupled run`_. + +Layer keys +`````````` + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``name`` + - text, required + - – + - Name of the layer in the messages and logs; unique in the section. + * - ``bottom`` + - raster, required + - m + - Elevation of the bottom of the layer. + * - ``initial_head`` + - raster, required + - m + - Head at the start of the simulation. + * - ``boundary`` + - raster, required + - – + - The MODFLOW ``IBOUND`` array of the layer: ``1`` active, ``0`` + inactive, ``-1`` constant head. A missing value is inactive. + * - ``laytype`` + - integer, required + - – + - BCF layer type. The tens digit is the interblock conductance average + (``0`` harmonic, ``1`` arithmetic, ``2`` logarithmic, ``3`` arithmetic + thickness and logarithmic conductivity); the units digit is ``LAYCON``: + ``0`` confined, ``1`` unconfined (top layer only), ``2`` and ``3`` + convertible. One of ``0``–``3``, ``10``–``13``, ``20``–``23``, + ``30``–``33``. + * - ``horizontal_conductivity`` + - raster, positive number, or ``{"map", "table"}`` + - m/day + - Horizontal hydraulic conductivity. The object form takes a nominal map + of classes (``map``) and a PCRaster lookup table (``table``) with one + ``class value`` row per class; the first row that matches a class is + the one read, as in any PCRaster lookup. + * - ``vertical_conductivity`` + - raster or positive number + - m/day + - Vertical hydraulic conductivity; MODFLOW's vertical leakance between + layers is computed from it and the layer thicknesses. + * - ``specific_storage`` + - raster, number ``>= 0`` or ``null`` + - – + - The confined storage coefficient of the layer, given to BCF as the + primary storage ``Sf1`` as it is (it is **not** multiplied by the + layer thickness). Read in transient runs by ``LAYCON`` 0, 2 and 3. + * - ``specific_yield`` + - raster, number in ``[0, 1]`` or ``null`` + - – + - Specific yield. Read in transient runs as ``Sf1`` by ``LAYCON`` 1 and + as the secondary storage ``Sf2`` by ``LAYCON`` 2 and 3. + * - ``compute_conductivity`` + - boolean, default ``true`` + - – + - With ``false`` the two conductivity inputs are given to MODFLOW as they + are, instead of the transmissivity and vertical leakance computed + from them (the ``COMPUTE`` flag of the extension's + ``setConductivity``); their units are then those of the MODFLOW arrays + they replace. + +A transient run needs the storage its layer types read: ``specific_storage`` +for ``LAYCON`` 0, ``specific_yield`` for ``LAYCON`` 1, both for ``LAYCON`` 2 +and 3. A steady-state run reads neither. + +Time discretization +``````````````````` + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``dis.nstp`` + - integer ``>= 1``, default ``1`` + - – + - Number of MODFLOW time steps in each stress period. With more than + one, a solver failure before the last of them ends the process instead + of raising an error; see `Limitations`_. + * - ``dis.tsmult`` + - number ``> 0``, default ``1.0`` + - – + - Multiplier of the length of successive time steps. + * - ``dis.steady_state`` + - boolean, default ``false`` + - – + - With ``true`` every stress period is steady state: no storage is read + and the ``storage`` output cannot be enabled. + +In a transient run the length of each period is updated to the days of its +month before it runs. The time unit (days) and the length unit (metres) are +fixed. + +Solver +`````` + +The solver is the preconditioned conjugate-gradient package (PCG), the only +one the coupling configures. Its keys are those of ``setPCG`` of the extension: + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``solver.mxiter`` + - integer ``>= 1``, default ``2000`` + - – + - Maximum number of outer iterations. + * - ``solver.iter1`` + - integer ``>= 1``, default ``20`` + - – + - Number of inner iterations. + * - ``solver.npcond`` + - ``1`` or ``2``, default ``1`` + - – + - Preconditioning: ``1`` modified incomplete Cholesky, ``2`` + polynomial. + * - ``solver.hclose`` + - number ``> 0``, default ``5.0`` + - m + - Head change criterion for convergence. + * - ``solver.rclose`` + - number ``> 0``, default ``3.0`` + - :raw-html:`m3day-1` + - Residual criterion for convergence. + * - ``solver.relax`` + - number ``> 0``, default ``1.0`` + - – + - Relaxation parameter, used with ``npcond`` 1. + * - ``solver.nbpol`` + - integer ``>= 0``, default ``2`` + - – + - Whether the estimate of the upper bound on the maximum eigenvalue is + 2.0 (PCG ``NBPOL``). + * - ``solver.damp`` + - number ``> 0``, default ``0.5`` + - – + - Damping factor. + +.. note:: + + The default criteria, ``hclose`` 5 m and ``rclose`` 3 + :raw-html:`m3day-1`, are coarse: they are the values + of the first applications of the coupling and they are kept as defaults, + not recommended. Tighten them for a study whose heads matter. + +The head MODFLOW gives a cell that becomes dry (``HDRY``) is fixed at +``-999.9``. + +Wetting +``````` + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``wetting.enabled`` + - boolean, default ``false`` + - – + - Whether dry cells may become wet again. + * - ``wetting.map`` + - raster, required when enabled + - m + - The ``WETDRY`` array: its absolute value is the wetting threshold; a + negative value lets only the cell below a dry cell wet it, a positive + one also the four horizontal neighbours, and ``0`` never wets the + cell. A missing value is ``0``. + * - ``wetting.layers`` + - list of layer numbers, or ``null`` (default) + - – + - The layers the map applies to; ``null`` means every layer whose + ``LAYCON`` is 1 or 3. + * - ``wetting.wetfct`` + - number ``> 0``, default ``1.0`` + - – + - Factor of the head a cell takes when it becomes wet. + * - ``wetting.iwetit`` + - integer ``>= 1``, default ``3`` + - – + - Wetting is attempted every ``iwetit`` outer iterations. + * - ``wetting.ihdwet`` + - ``0`` or ``1``, default ``0`` + - – + - Equation of the head of a cell that becomes wet: ``0`` + :math:`h = BOT + WETFCT (h_n - BOT)`, ``1`` + :math:`h = BOT + WETFCT \cdot THRESH`. + +MODFLOW-2005 reads ``WETDRY`` only for layers whose ``LAYCON`` is 1 or 3, so +every listed layer must have one of those types: a model whose lower layers +are ``LAYCON`` 2 needs ``laytype`` 3 on them to be rewetted. + +Rivers, general-head boundaries and drains +`````````````````````````````````````````` + +Each package has ``enabled`` (default ``false``) and a list of ``entries``; +an enabled package needs at least one entry. An entry applies the same maps +to every layer of its ``layers`` list, and a layer may appear in at most one +entry of a package. The stress data are read and given to MODFLOW once, when +the run starts, and hold for every stress period. + +A cell of an entry, in a listed layer, is a cell with a positive conductance +(inside the ``mask`` for a river) that is active in that layer (``boundary`` +not ``0``); the conductance is zero everywhere else. + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``river.entries[].layers`` + - list of layer numbers + - – + - The layers the river cells belong to. + * - ``river.entries[].stage`` + - raster + - m + - River stage. + * - ``river.entries[].bottom`` + - raster + - m + - Elevation of the riverbed bottom. + * - ``river.entries[].conductance`` + - raster or positive number + - :raw-html:`m2day-1` + - Riverbed conductance of each river cell. A number needs ``mask``. + * - ``river.entries[].mask`` + - raster, optional + - – + - The river cells: the cells where it is positive. With a conductance + raster, it restricts the river to those cells. + * - ``ghb.entries[].layers`` + - list of layer numbers + - – + - The layers of the boundary. + * - ``ghb.entries[].head`` + - raster + - m + - Head of the external source. + * - ``ghb.entries[].conductance`` + - raster + - :raw-html:`m2day-1` + - Boundary conductance. + * - ``drain.entries[].layers`` + - list of layer numbers + - – + - The layers of the drains. + * - ``drain.entries[].elevation`` + - raster + - m + - Drain elevation. + * - ``drain.entries[].conductance`` + - raster + - :raw-html:`m2day-1` + - Drain conductance. + +.. warning:: + + MODFLOW defines a boundary cell by its layer, row and column, so an entry + listing several layers creates one boundary cell per layer, each with the + full conductance of the map. A general-head boundary on the three layers + of a column therefore has **three times** the conductance of the map + between the aquifer and its source. Divide the conductance map by the + number of layers when the map holds the conductance of the whole column. + +A river whose cells lie at different depths can be split into several entries, +each with its own ``mask`` and layer. + +The root-depth coupling +``````````````````````` + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``coupling.dynamic_root_depth.enabled`` + - boolean, default ``false`` + - – + - Whether the water table restricts the roots of the vegetation, see + `Roots and the water table`_. + * - ``coupling.dynamic_root_depth.minimum_depth_table`` + - lookup table, required when enabled + - cm + - Minimum root depth of each soil class (the classes of + ``RASTERS.soil``), in the units of the rootzone depth table. + * - ``coupling.dynamic_root_depth.water_table.method`` + - ``"highest_unconfined"`` (default), ``"highest_active_head"`` or + ``"layer"`` + - – + - Which head is the water table, cell by cell. + * - ``coupling.dynamic_root_depth.water_table.layer`` + - layer number, only with ``"layer"`` + - – + - The layer whose head is the water table. + +The three methods read the heads of the step, leaving out the cells that are +dry or inactive in their layer: + +- ``highest_active_head`` takes the first head from the top down; +- ``highest_unconfined`` does the same over the layers whose ``LAYCON`` is 1, + 2 or 3, and skips a convertible layer (``LAYCON`` 2 or 3) where its head is + above the top of the layer, that is, where the layer is confined; it needs a + layer of one of those types; +- ``layer`` takes the head of ``water_table.layer``. + +Output keys +``````````` + +.. list-table:: + :header-rows: 1 + :widths: 22 22 12 44 + + * - Key + - Type + - Unit + - Description + * - ``output.heads`` + - boolean, default ``true`` + - – + - Head of every layer. + * - ``output.river_leakage`` + - boolean, default ``true`` + - – + - River exchange, in both directions and net. + * - ``output.storage`` + - boolean, default ``false`` + - – + - Storage flow of every layer; transient runs only. + * - ``output.drain_flow`` + - boolean, default ``false`` + - – + - Drain flow of every drain layer. + * - ``output.root_depth`` + - boolean, default ``false`` + - – + - The water table and root depth of the root-depth coupling; written + only when that coupling is enabled. + +Rules the configuration and the inputs must follow +-------------------------------------------------- + +When the configuration is loaded +```````````````````````````````` + +An enabled section is refused when it breaks one of these rules; every +violation is listed in one message: + +- ``top`` and at least one layer are given, at most 9 layers, and the layer + names are unique; +- ``LAYCON`` 1 appears only on layer 1, as MODFLOW-2005 requires; +- ``river.enabled`` is ``true``, every enabled package has an entry, every + layer number is a layer of the section and appears at most once per + package, and a numeric river conductance comes with a ``mask``; +- a transient run gives the storage each layer type reads (see + `Layer keys`_), and a steady-state run does not ask for the ``storage`` + output; +- enabled wetting gives ``map``, and each of its layers has ``LAYCON`` 1 or 3 + (with ``layers`` ``null``, at least one layer has); +- an enabled root-depth coupling gives ``minimum_depth_table``; + ``water_table.layer`` is given with the ``layer`` method, and only with it. + +Before every coupled run +```````````````````````` + +Always, even with ``-s``, and blocking: every file the coupled run reads +exists and is not empty (the files of a disabled package, of disabled wetting +and of a disabled root-depth coupling are not read, so not checked), and the +MODFLOW runtime is available (see `Requirements`_). + +With the validation of the inputs, which ``-s`` skips, the content is checked. +Blocking: + +- a raster that cannot be read or does not share the geometry of the clone; +- a boundary value other than ``-1``, ``0`` and ``1``; +- the model top or a bottom missing on a column active in any layer, or a + bottom not strictly below the top of its layer there (the message names the + layer and the first row and column); +- an initial head, conductivity or (in a transient run) storage raster + missing on an active cell of its layer; +- an initial head below the layer bottom in every active cell of the layer; +- an initial head equal to a MODFLOW no-data marker (``-888``, ``-999``, + ``-999.9``, ``-999.99`` or ``-9999``) in an active cell: the extension + would read it as a head, start the cell dry and carry the value into the + head outputs, so give the cell a head or make it inactive; +- a conductivity class map in PCRaster format that is not nominal, or a + class on an active cell that the lookup table does not give a positive + value; +- an enabled package layer without any cell (the extension would end the + process on it), or a stage, bottom, head or elevation missing on a cell of + the package; +- with the root-depth coupling, a minimum root depth table that does not + cover the soil classes, or a value that is not in + :math:`(0, Z_r]`, :math:`Z_r` being the rootzone depth of the class. + +Non-blocking warnings: + +- an initial head below the layer bottom in some active cells (the count is + given): MODFLOW starts with those cells dry; +- river cells whose riverbed bottom lies outside the elevation interval of + their layer, reported as ``N of M river cells of layer k have their bed + outside the layer's elevation interval``, with how many are below the + layer bottom and how many above the layer top. Moving them to another + layer, with a second river entry, is a modelling decision the run does + not take. + +Outputs of a coupled run +------------------------ + +The diagnostics are raster series, one map per step, written in the raster +formats of the run (``RASTER_FILE_FORMAT``) with the naming of the other +output variables (``mfh10000.001`` in PCRaster format, ``mfh1000001.tif`` in +GeoTIFF for the head of layer 1 at step 1). They are selected by the +``output`` keys of the section, not by ``GENERATE_FILE``, and no time series +is written for them, so a run that enables no raster format writes none of +them. + +.. list-table:: + :header-rows: 1 + :widths: 18 22 14 46 + + * - Prefix + - Written when + - Unit + - Content + * - ``mfh`` + - ``output.heads`` + - m + - Head of layer ``n``. Dry cells have no value. + * - ``mfaq2rv`` + - ``output.river_leakage`` + - :raw-html:`m3day-1` + - Leakage from the aquifer to the river, summed over the river layers; + ``bfw`` is this flux as a depth. + * - ``mfrv2aq`` + - ``output.river_leakage`` + - :raw-html:`m3day-1` + - Leakage from the river to the aquifer. + * - ``mfrvnet`` + - ``output.river_leakage`` + - :raw-html:`m3day-1` + - Net river leakage, positive into the aquifer. + * - ``mfst`` + - ``output.storage`` (transient) + - :raw-html:`m3day-1` + - Storage flow of layer ``n``. + * - ``mfdrn`` + - ``output.drain_flow`` + - :raw-html:`m3day-1` + - Drain flow of layer ``n``, negative out of the aquifer; one series per + layer of the drain entries. + * - ``mfwt`` + - ``output.root_depth`` and the root-depth coupling + - m + - Water-table head of the step, the one the next step's roots see. + * - ``mfgwd`` + - ``output.root_depth`` and the root-depth coupling + - cm + - Depth of the previous step's water table below the terrain; no value + at step 1. + * - ``mfzr`` + - ``output.root_depth`` and the root-depth coupling + - cm + - Effective root depth of the vegetation. + * - ``mfzfrac`` + - ``output.root_depth`` and the root-depth coupling + - – + - Effective root depth as a fraction of the rootzone depth. + +The ``bfw`` output variable, its raster series and its time series, carries +the baseflow of the coupling, the aquifer-to-river leakage in millimetres per +step (see `Baseflow`_). + +The run directory +----------------- + +MODFLOW writes its input and result files (``pcrmf.*``, ``fort.*``) into +``/modflow``, which the run creates when it starts and +removes after the last step. The run owns that directory: it refuses to start +when the directory already exists, so that it never removes files it did not +write, and removes it again when MODFLOW cannot start. + +A run that fails during the simulation keeps the directory, with the MODFLOW +listing file ``pcrmf.lst``, for inspection; move or remove it before running +into the same output directory again. + +In a calibration each evaluation runs in its own temporary output directory +under ``--temp-dir``, so its MODFLOW directory, and the conductivity tables of +its candidate, are removed with it. + +Roots and the water table +------------------------- + +.. warning:: + + The root-depth coupling is experimental and off by default. It changes the + evapotranspiration of the vegetated area. + +With ``coupling.dynamic_root_depth.enabled``, the water table of a step +restricts the roots of the vegetation at the next step. The depth of the water +table is measured from the terrain, the DEM of the configuration +(``RASTERS.dem``), not from the model top: + +.. math:: + :label: groundwater-depth + :nowrap: + + \[D_{GW} = \max\left(100 \left(z_{DEM} - h_{WT}\right),\ 0\right)\] + +.. math:: + :label: groundwater-root-depth + :nowrap: + + \[Z_{r,eff} = \min\left(Z_r,\ \max\left(Z_{r,min},\ D_{GW}\right)\right), \quad f_r = \frac{Z_{r,eff}}{Z_r}\] + +where: + +- :math:`D_{GW}` – depth of the water table below the terrain [cm]; +- :math:`z_{DEM}` – terrain elevation [m]; +- :math:`h_{WT}` – water-table head of the previous step [m], chosen by + ``water_table.method``; +- :math:`Z_r` – rootzone depth of the soil class [cm]; +- :math:`Z_{r,min}` – minimum root depth of the soil class, from + ``minimum_depth_table`` [cm]; +- :math:`Z_{r,eff}` – effective root depth [cm]; +- :math:`f_r` – fraction of the rootzone the roots reach [-]. + +At the first step, and wherever the previous water table has no value, the +roots reach the whole rootzone (:math:`f_r = 1`). The water stress coefficient +of the vegetated area is then computed on the soil moisture, the wilting point +and the field capacity of the rootzone multiplied by :math:`f_r`; the bare +soil keeps the coefficient of the whole rootzone, and the soil water balance +itself is not changed. + +Calibrating a coupled model +--------------------------- + +``rubem calibrate`` runs the coupled model at every evaluation whenever the +configuration enables MODFLOW; no option is needed. The storage of the layers, +the conductivity of the classes of a lookup table and the numeric river +conductances can join the search, by name, when ``--bound`` or ``--fix`` names +them. :ref:`calibration:MODFLOW parameters` lists the names and the rules. + +Limitations +----------- + +- **Solver failures.** A stress period that does not converge raises an error + that names the period and the listing file, and the run stops, keeping its + run directory; in a calibration the evaluation fails and is ranked behind + every other. That holds when the solver fails at the last time step of the + period, which is always the case with the default ``dis.nstp`` of 1. With + more time steps, a failure at an earlier one stops MODFLOW before it writes + the heads of the period, and the PCRaster MODFLOW extension then ends the + whole process (``Can not open head value result file``) instead of + returning an error; the run directory, with ``pcrmf.lst``, is kept. In a + calibration that ends the worker process, and with it the calibration, + which is why ``rubem calibrate`` warns when ``dis.nstp`` is above 1. +- The MODFLOW packages are DIS, BAS, BCF, RCH, RIV, GHB, DRN and the PCG + solver, and no other: no well package (WEL), no other flow package or + solver. +- The stress packages (RIV, GHB, DRN) are static: their maps are read once and + hold for the whole simulation. +- Days and metres only; square cells of side ``GRID.grid``. +- At most 9 layers. +- The MODFLOW diagnostics are raster series only: no time series of them is + written, and the evaluations of a calibration, which write no raster, do not + write them either. Their seven-character prefixes (``mfaq2rv``, + ``mfrv2aq``, ``mfrvnet``, ``mfzfrac``) leave room for 999 steps in GeoTIFF + names and 9,999 in PCRaster names. +- The conductance of a general-head boundary, a drain or a river entry that + lists several layers is given to each of them, so it adds up across the + layers (see the warning under + `Rivers, general-head boundaries and drains`_). +- Wetting applies one ``WETDRY`` map to its layers, which must be ``LAYCON`` 1 + or 3. +- The river-to-aquifer leakage adds water to the aquifer that no RUBEM flux + loses. + +References +---------- + +- PCRaster MODFLOW extension, version 4.4.2: + `overview `__, + `layer numbering (DIS) `__, + `BCF `__, + `RIV `__, + `RCH `__, + `GHB `__, + `DRN `__, + `solvers `__. +- Online guide to MODFLOW-2005: + `index `__, + `BCF `__, + `RIV `__, + `GHB `__, + `DRN `__, + `PCG `__. + +.. [HARBAUGH2005] Harbaugh, A.W. (2005). MODFLOW-2005, the U.S. Geological + Survey modular ground-water model — the Ground-Water Flow Process. U.S. + Geological Survey Techniques and Methods 6-A16. + https://pubs.usgs.gov/tm/2005/tm6A16/ diff --git a/doc/source/index.rst b/doc/source/index.rst index 36615ff..ab48203 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -33,6 +33,7 @@ All development activity is coordinated via the `RUBEM Github page ` +Optional boolean value. If enabled, this option allows the generation of Baseflow (BFW) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` When the :ref:`MODFLOW coupling ` is enabled, the baseflow is the leakage from the aquifer to the river cells of the step, as a depth over the cell. :ref:`See more. ` .. code-block:: json @@ -1069,6 +1069,284 @@ Optional boolean value. If enabled, this option allows the generation of Accumul }, } +MODFLOW Groundwater Coupling +---------------------------- + +The optional ``MODFLOW`` section (``modflow`` in configuration format 1.0) +couples the model to MODFLOW-2005 through the PCRaster MODFLOW extension: the +recharge of each step feeds the groundwater model and the leakage from the +aquifer to the rivers becomes the baseflow. Without the section, or with +``enabled`` set to ``false``, the run is not coupled and the saturated zone is +the lumped reservoir of the model. The coupling, the rules the inputs must +follow, the outputs and the limitations are documented in +:doc:`Groundwater Coupling with MODFLOW `; this section lists the +keys. Relative paths are anchored on the directory of the configuration file, +lengths are in metres and times in days, and a key the section does not know is +refused. + +MODFLOW Section +``````````````` + +Optional boolean value ``enabled`` (default ``false``) that turns the coupling +on, and mandatory path ``top`` to a raster of the elevation of the model top +[m] when it is on. A section given with ``enabled`` set to ``false`` is +reported as ignored. The coupling needs the PCRaster MODFLOW extension and the +``mf2005`` executable, both installed by the conda-forge ``pcraster`` package; +when either is missing, or a MODFLOW input file does not exist, the validation +reports a blocking problem, even with ``-s``. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "enabled": true, + "top": "/Dataset/UIGCRB/input/modflow/top_model.map", + }, + } + +MODFLOW Layers +`````````````` + +Mandatory list ``layers`` of one to nine layers, listed **from the top down**: +layer 1 is the top layer, and that number is the one every other key uses. +Each layer has a unique ``name``; the paths ``bottom`` (elevation of the bottom +of the layer [m]), ``initial_head`` [m] and ``boundary`` (the MODFLOW +``IBOUND`` raster: ``1`` active, ``0`` inactive, ``-1`` constant head); the BCF +``laytype`` (units digit ``LAYCON``: ``0`` confined, ``1`` unconfined, top +layer only, ``2`` and ``3`` convertible; tens digit the interblock averaging); +``horizontal_conductivity`` [m/day], a raster path, a positive number or an +object with a nominal class ``map`` and a lookup ``table``; +``vertical_conductivity`` [m/day], a raster path or a positive number; +``specific_storage`` (the confined storage coefficient, given to MODFLOW as it +is) and ``specific_yield`` [dimensionless quantity], each a raster path or a +number; and ``compute_conductivity`` (default ``true``). A transient run needs +``specific_storage`` for ``LAYCON`` 0, ``specific_yield`` for ``LAYCON`` 1 and +both for ``LAYCON`` 2 and 3. Numbers are written without quotes. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "layers": [ + { + "name": "upper", + "bottom": "/Dataset/UIGCRB/input/modflow/bottom1.map", + "initial_head": "/Dataset/UIGCRB/input/modflow/head1.map", + "boundary": "/Dataset/UIGCRB/input/modflow/bound.map", + "laytype": 1, + "horizontal_conductivity": { + "map": "/Dataset/UIGCRB/input/modflow/kh_classes1.map", + "table": "/Dataset/UIGCRB/input/modflow/kh1.tbl", + }, + "vertical_conductivity": "/Dataset/UIGCRB/input/modflow/kv1.map", + "specific_yield": 0.15, + "specific_storage": 1e-6, + }, + { + "name": "lower", + "bottom": "/Dataset/UIGCRB/input/modflow/bottom2.map", + "initial_head": "/Dataset/UIGCRB/input/modflow/head2.map", + "boundary": "/Dataset/UIGCRB/input/modflow/bound.map", + "laytype": 2, + "horizontal_conductivity": 0.1, + "vertical_conductivity": 0.02, + "specific_yield": 0.15, + "specific_storage": 1e-6, + }, + ], + }, + } + +MODFLOW Time Discretization +``````````````````````````` + +Optional object ``dis``: ``nstp``, the number of MODFLOW time steps in each +stress period (default ``1``); ``tsmult``, their length multiplier (default +``1.0``); and ``steady_state`` (default ``false``). One RUBEM step is one +stress period, whose length is the number of days of its month. With ``nstp`` +above 1, a solver failure before the last time step of a period ends the +process instead of raising an error. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "dis": {"nstp": 5, "tsmult": 1.0, "steady_state": false}, + }, + } + +MODFLOW Solver +`````````````` + +Optional object ``solver`` with the settings of the preconditioned +conjugate-gradient solver (PCG), the only one available: ``mxiter`` (default +``2000``), ``iter1`` (``20``), ``npcond`` (``1`` or ``2``, default ``1``), +``hclose`` [m] (``5.0``), ``rclose`` [:raw-html:`m3day-1`] +(``3.0``), ``relax`` (``1.0``), ``nbpol`` (``2``) and ``damp`` (``0.5``). The +default convergence criteria are coarse. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "solver": {"hclose": 0.01, "rclose": 1.0}, + }, + } + +MODFLOW Wetting +``````````````` + +Optional object ``wetting`` for the rewetting of dry cells: ``enabled`` +(default ``false``); ``map``, the path to the ``WETDRY`` raster [m], mandatory +when enabled; ``layers``, the layers it applies to (default ``null``: every +layer whose ``LAYCON`` is 1 or 3; a listed layer must be of one of those +types); ``wetfct`` (``1.0``), ``iwetit`` (``3``) and ``ihdwet`` (``0`` or +``1``, default ``0``). :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "wetting": { + "enabled": true, + "map": "/Dataset/UIGCRB/input/modflow/wet.map", + "layers": [1], + }, + }, + } + +MODFLOW Rivers +`````````````` + +Mandatory object ``river`` when the section is enabled, with ``enabled`` set +to ``true``: the leakage from the aquifer to the river cells is the baseflow +of the coupled run. Each of its ``entries`` lists its ``layers`` and the paths +``stage`` [m] and ``bottom`` (riverbed bottom) [m]; ``conductance`` +[:raw-html:`m2day-1`] is a raster path or a positive +number, which then needs ``mask``, the path to a raster whose positive cells +are the river cells. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "river": { + "enabled": true, + "entries": [ + { + "layers": [1], + "stage": "/Dataset/UIGCRB/input/modflow/riv_stage.map", + "bottom": "/Dataset/UIGCRB/input/modflow/riv_bot.map", + "conductance": 0.387, + "mask": "/Dataset/UIGCRB/input/modflow/riv_cells.map", + }, + ], + }, + }, + } + +MODFLOW General-Head Boundaries +``````````````````````````````` + +Optional object ``ghb`` (default disabled) whose ``entries`` list their +``layers`` and the raster paths ``head`` [m] and ``conductance`` +[:raw-html:`m2day-1`]. The same maps apply to every +listed layer, so the boundary conductance of a column adds up over its +layers. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "ghb": { + "enabled": true, + "entries": [ + { + "layers": [1, 2], + "head": "/Dataset/UIGCRB/input/modflow/ghb_head.map", + "conductance": "/Dataset/UIGCRB/input/modflow/ghb_cond.map", + }, + ], + }, + }, + } + +MODFLOW Drains +`````````````` + +Optional object ``drain`` (default disabled) whose ``entries`` list their +``layers`` and the raster paths ``elevation`` [m] and ``conductance`` +[:raw-html:`m2day-1`]. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "drain": { + "enabled": true, + "entries": [ + { + "layers": [1], + "elevation": "/Dataset/UIGCRB/input/modflow/drn_elev.map", + "conductance": "/Dataset/UIGCRB/input/modflow/drn_cond.map", + }, + ], + }, + }, + } + +MODFLOW Root-Depth Coupling +``````````````````````````` + +Optional object ``coupling.dynamic_root_depth``, experimental and disabled by +default, with which the water table of a step restricts the roots of the +vegetation at the next one: ``enabled``; ``minimum_depth_table``, the path to +a lookup table of the minimum root depth [cm] of each soil class, mandatory +when enabled, with values in :math:`(0, Z_r]`; and ``water_table``, whose +``method`` is ``highest_unconfined`` (default), ``highest_active_head`` or +``layer``, the last one with the ``layer`` number. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "coupling": { + "dynamic_root_depth": { + "enabled": true, + "minimum_depth_table": "/Dataset/UIGCRB/input/txt/soil/Dpz_min.txt", + "water_table": {"method": "highest_active_head"}, + }, + }, + }, + } + +MODFLOW Outputs +``````````````` + +Optional object ``output`` of boolean values selecting the MODFLOW +diagnostics written as raster series, in the raster formats of the run: +``heads`` (default ``true``, ``mfh`` [m]), ``river_leakage`` (default +``true``, ``mfaq2rv``, ``mfrv2aq`` and ``mfrvnet`` +[:raw-html:`m3day-1`]), ``storage`` (default ``false``, +``mfst``, transient runs only), ``drain_flow`` (default ``false``, +``mfdrn``) and ``root_depth`` (default ``false``, ``mfwt``, ``mfgwd``, +``mfzr`` and ``mfzfrac``, with the root-depth coupling). No time series is +written for them. :ref:`See more. ` + +.. code-block:: json + + { + "MODFLOW": { + "output": { + "heads": true, + "river_leakage": true, + "storage": false, + "drain_flow": false, + "root_depth": false, + }, + }, + } + Configuration File Template --------------------------- @@ -1161,7 +1439,38 @@ Configuration File Template "map_raster_series": true, "tiff_raster_series": true, }, - } + "MODFLOW": { + "enabled": false, + "top": "/Dataset/UIRB/input/modflow/top.map", + "layers": [ + { + "name": "aquifer", + "bottom": "/Dataset/UIRB/input/modflow/bottom.map", + "initial_head": "/Dataset/UIRB/input/modflow/head.map", + "boundary": "/Dataset/UIRB/input/modflow/bound.map", + "laytype": 1, + "horizontal_conductivity": 0.5, + "vertical_conductivity": 0.1, + "specific_yield": 0.15, + }, + ], + "river": { + "enabled": true, + "entries": [ + { + "layers": [1], + "stage": "/Dataset/UIRB/input/modflow/riv_stage.map", + "bottom": "/Dataset/UIRB/input/modflow/riv_bot.map", + "conductance": "/Dataset/UIRB/input/modflow/riv_cond.map", + }, + ], + }, + }, + } + +The ``MODFLOW`` section is optional. The template shows it disabled, which the +validation reports as ignored: remove it, or set ``enabled`` to ``true`` to +couple the run to MODFLOW, see :ref:`userguide:MODFLOW Groundwater Coupling`. ------------------ @@ -1246,9 +1555,16 @@ Use ``-h`` or ``--help`` to get a brief description of each command and its argu --temp-dir Parent of the per-evaluation output directories. --bound Narrow the range of one parameter: - NAME=MIN:MAX. + NAME=MIN:MAX. A MODFLOW parameter + (modflow.layers..specific_yield, + modflow.layers..specific_storage, + modflow.layers..kh., + modflow.river..conductance) is searched + only when bounded; see the calibration page + of the documentation. --fix Keep one parameter out of the search: - NAME=VALUE. + NAME=VALUE; MODFLOW parameters use the names + of --bound. --stations Station ids of the objective, comma- separated (default: every shared station). --init @@ -1377,6 +1693,9 @@ another grid) is reached by the simulation. Failures that are not validation problems still stop the run before it starts: a configuration file that does not match the schema, a lookup table file that is not there, no raster format enabled, or a DEM, clone and georeference that do not share their geometry. +With an enabled :ref:`MODFLOW section `, +``-s`` still checks that its input files exist and that the MODFLOW runtime is +installed, since the coupled run cannot start without them. .. code-block:: console diff --git a/rubem/_deps.py b/rubem/_deps.py index d6e846e..bc7f1cf 100644 --- a/rubem/_deps.py +++ b/rubem/_deps.py @@ -1,6 +1,13 @@ """Diagnostics for the conda-only native runtime dependencies.""" import importlib.util +import logging +import os +import shutil +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) _CONDA_ONLY_DEPENDENCIES = ("pcraster", "osgeo") _ENVIRONMENT_YML_URL = "https://github.com/LabSid-USP/RUBEM/blob/main/environment.yml" @@ -119,3 +126,96 @@ def require_calibration_deps() -> None: missing = missing_calibration_deps() if missing: raise SystemExit(calibration_deps_message(missing)) + + +_GROUNDWATER_EXTENSION = "pcraster._pcraster_modflow" +_MF2005 = "mf2005" + + +def resolve_mf2005() -> Path | None: + """Return the MODFLOW-2005 executable the PCRaster MODFLOW extension launches. + + The extension starts ``mf2005`` from PATH. The conda-forge ``pcraster`` + package installs it next to the interpreter (``/bin/mf2005``, or + ``/Library/bin/mf2005.exe`` on Windows), which is not on PATH when + the environment is used without being activated, so that location is the + fallback. + + :returns: The executable, or ``None`` when neither PATH nor the prefix has it. + :rtype: pathlib.Path | None + """ + found = shutil.which(_MF2005) + if found is not None: + return Path(found) + prefix = Path(sys.prefix) + for candidate in (prefix / "bin" / _MF2005, prefix / "Library" / "bin" / f"{_MF2005}.exe"): + if candidate.is_file(): + return candidate + return None + + +def missing_groundwater_deps() -> list[str]: + """Return what the MODFLOW coupling needs and cannot find. + + The PCRaster MODFLOW extension (``pcraster._pcraster_modflow``, which + provides ``pcraster.initialise``) and the ``mf2005`` executable, see + :func:`resolve_mf2005`. As in :func:`missing_runtime_deps`, an extension + whose lookup fails (looking up a submodule imports ``pcraster``) counts as + missing. + + :returns: The missing names, in the order they are checked. + :rtype: list[str] + """ + missing = [] + try: + found = importlib.util.find_spec(_GROUNDWATER_EXTENSION) is not None + except (ImportError, ValueError): + found = False + if not found: + missing.append(_GROUNDWATER_EXTENSION) + if resolve_mf2005() is None: + missing.append(_MF2005) + return missing + + +def groundwater_deps_message(missing: list[str] | None = None) -> str: + """Return the installation guidance for the missing MODFLOW runtime. + + :param missing: The missing names, see :func:`missing_groundwater_deps`. + Defaults to ``None``, which names both. + :type missing: list[str] | None + :rtype: str + """ + names = [_GROUNDWATER_EXTENSION, _MF2005] if missing is None else missing + return ( + "The MODFLOW coupling needs the PCRaster MODFLOW extension " + f"({_GROUNDWATER_EXTENSION}) and the MODFLOW-2005 executable ({_MF2005}), both " + f"shipped by the conda-forge pcraster package; not found: {', '.join(names)}. " + "Install pcraster from conda-forge (the pinned specification is environment.yml: " + f"{_ENVIRONMENT_YML_URL}), or put {_MF2005} on PATH." + ) + + +def ensure_mf2005_on_path() -> Path | None: + """Put the directory of :func:`resolve_mf2005` on PATH when it is not there. + + The extension launches ``mf2005`` from PATH, and spawned calibration + workers inherit the environment of the parent, so the directory is + prepended once, in the parent, and logged. + + :returns: The executable, or ``None`` when it cannot be found (PATH is + then left unchanged). + :rtype: pathlib.Path | None + """ + found = shutil.which(_MF2005) + if found is not None: + return Path(found) + executable = resolve_mf2005() + if executable is None: + return None + directory = str(executable.parent) + path = os.environ.get("PATH", "") + if directory not in path.split(os.pathsep): + os.environ["PATH"] = f"{directory}{os.pathsep}{path}" if path else directory + logger.info("Added %s to PATH so that MODFLOW finds %s.", directory, _MF2005) + return executable diff --git a/rubem/_dynamic_model.py b/rubem/_dynamic_model.py index b14957f..16599b1 100644 --- a/rubem/_dynamic_model.py +++ b/rubem/_dynamic_model.py @@ -1,9 +1,11 @@ import logging import os +import shutil import warnings from calendar import monthrange from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING import numpy as np import pcraster as pcr @@ -19,8 +21,14 @@ from .file._timeoutput import TimeoutputTimeseriesAdapter from .hydrological_processes import Evapotranspiration, Interception, Soil, SurfaceRunoff +if TYPE_CHECKING: + from .hydrological_processes._modflow import ModflowGroundwater, ModflowStepResult + MISSING_VALUE_DEFAULT = -9999 +MODFLOW_RUN_DIRECTORY = "modflow" +"""Directory of the output directory MODFLOW runs in; removed after the last step.""" + class RainfallRunoffBalanceEnhancedModel(pcrfw.DynamicModel): """Rainfall-Runoff Balance Enhanced Model. @@ -85,6 +93,16 @@ def __init__(self, config: ModelConfiguration): self.current_cell_total_discharge = None self.accumulated_cell_total_discharge = None self.current_runoff = None + # MODFLOW coupling (only when the configuration enables it). + self.modflow: ModflowGroundwater | None = None + self.modflow_run_dir: Path | None = None + self.surface_elevation = None + self.soil_rootzone_depth_min = None + self.previous_water_table_head = None + # Root-depth coupling of the current step: computed, reported and then released. + self.groundwater_depth_cm = None + self.effective_root_depth = None + self.root_depth_fraction = None @property def soil_moistute_content_wilting_point(self): @@ -232,8 +250,58 @@ def initial(self): self.initial_cell_total_flow = pcrfw.scalar(0) self.previous_cell_total_flow = pcrfw.scalar(0) + if self.config.modflow_enabled: + self.__initial_modflow(soil) + self.__release_initial_state() + def __initial_modflow(self, soil: Field) -> None: + """Start MODFLOW in ``/modflow`` and prepare the root-depth coupling. + + The module is imported here so that a run without MODFLOW never loads it. + The run owns its directory: it removes it after the last step, so it + refuses one that already exists, and removes the one it has just + created when MODFLOW cannot start. + + :param soil: Soil classes of the run. + :type soil: Field + :raises RuntimeError: If ``/modflow`` already exists. + """ + from .hydrological_processes._modflow import ModflowGroundwater + + settings = self.config.modflow + self.modflow_run_dir = Path(self.config.output_directory.path) / MODFLOW_RUN_DIRECTORY + try: + self.modflow_run_dir.mkdir(parents=True, exist_ok=False) + except FileExistsError: + raise RuntimeError( + f"The MODFLOW run directory {self.modflow_run_dir} already exists (the listing " + "of a failed run, or files of yours); the run removes it at the end, so move " + "or remove it before the run." + ) from None + self.logger.info("Initializing MODFLOW groundwater module in %s...", self.modflow_run_dir) + start = self.config.simulation_period.start_date + try: + self.modflow = ModflowGroundwater( + settings, self.config.grid.area, self.modflow_run_dir, logger=self.logger + ) + self.modflow.initialize(monthrange(start.year, start.month)[1]) + except BaseException: + # Nothing has run yet, so the directory holds nothing to inspect. + self.modflow = None + shutil.rmtree(self.modflow_run_dir, ignore_errors=True) + raise + + if settings.coupling.dynamic_root_depth.enabled: + self.logger.info("Reading minimum soil root zone depth (Dpz_min)...") + # The water-table depth is measured from the terrain. + self.surface_elevation = self.dem + self.soil_rootzone_depth_min = self.__lookup_wrapper( + file_path=settings.coupling.dynamic_root_depth.minimum_depth_table, + lookup_value=soil, + lookup_func=pcrfw.lookupscalar, + ) + def __release_initial_state(self): """Drop the rasters the dynamic section never reads again. @@ -349,6 +417,9 @@ def dynamic(self): else: current_lai_max = self.config.constants.leaf_area_interception_max + if self.__root_depth_coupled: + self.__update_root_depth() + self.logger.debug("Interception") current_reflectances_simple_ratio = Interception.get_reflectances_simple_ratio(current_ndvi) current_fpar = Interception.get_fpar( @@ -391,10 +462,23 @@ def dynamic(self): self.soil_moisture_content_field_capacity, ) ) + if self.root_depth_fraction is None: + vegetation_water_stress_coef = water_stress_coef + else: + # The roots reach only a fraction of the rootzone: the vegetation + # sees the storages of that fraction, the bare soil the whole of + # them. The soil water balance itself is not changed. + vegetation_water_stress_coef = pcr.scalar( + Evapotranspiration.get_water_stress_coef_et_vegetated_area( + self.current_soil_moist_content * self.root_depth_fraction, + self.soil_moisture_content_wilting_point * self.root_depth_fraction, + self.soil_moisture_content_field_capacity * self.root_depth_fraction, + ) + ) # Vegetated area real_et_vegetated_area = Evapotranspiration.get_et_vegetated_area( - current_potential_evapotranspiration, current_crop_coef, water_stress_coef + current_potential_evapotranspiration, current_crop_coef, vegetation_water_stress_coef ) # Impervious area @@ -491,14 +575,25 @@ def dynamic(self): self.logger.debug("Baseflow") - self.current_baseflow = Soil.get_baseflow( - self.previous_baseflow, - self.config.calibration_parameters.alpha_gw, - self.current_recharge, - self.current_soil_sat_zone_storage, - self.baseflow_threshold, - ) - self.previous_baseflow = self.current_baseflow + modflow_result = None + if self.modflow is None: + self.current_baseflow = Soil.get_baseflow( + self.previous_baseflow, + self.config.calibration_parameters.alpha_gw, + self.current_recharge, + self.current_soil_sat_zone_storage, + self.baseflow_threshold, + ) + self.previous_baseflow = self.current_baseflow + else: + self.logger.debug("Running MODFLOW...") + modflow_result = self.modflow.run_step( + self.current_recharge, monthrange(current_date.year, current_date.month)[1] + ) + # The aquifer-to-river leakage replaces the saturated-zone reservoir. + self.current_baseflow = modflow_result.baseflow_mm + # The head of this step drives the roots of the next one. + self.previous_water_table_head = modflow_result.water_table_head self.logger.debug("Soil Balance") self.current_soil_moist_content = Soil.get_actual_soil_moist_cont( @@ -512,11 +607,13 @@ def dynamic(self): open_water_area_fraction, self.soil_moist_content_sat_point, ) - self.current_soil_sat_zone_storage = Soil.get_actual_water_cont_sat_zone( - self.previous_soil_sat_zone_storage, self.current_recharge, self.current_baseflow - ) + if self.modflow is None: + # MODFLOW holds the saturated zone when it is coupled. + self.current_soil_sat_zone_storage = Soil.get_actual_water_cont_sat_zone( + self.previous_soil_sat_zone_storage, self.current_recharge, self.current_baseflow + ) + self.previous_soil_sat_zone_storage = self.current_soil_sat_zone_storage self.previous_soil_moist_content = self.current_soil_moist_content - self.previous_soil_sat_zone_storage = self.current_soil_sat_zone_storage self.logger.debug("Runoff") self.current_cell_total_discharge = ( @@ -539,8 +636,51 @@ def dynamic(self): self.previous_cell_total_flow = self.current_runoff self.logger.debug("Exporting variables to files") - self.__current_step_report() + self.__current_step_report(modflow_result) self.__release_step_state() + if self.modflow is not None and current_timestep == self.config.simulation_period.last_step: + self.__finish_modflow() + + @property + def __root_depth_coupled(self) -> bool: + return ( + self.config.modflow_enabled and self.config.modflow.coupling.dynamic_root_depth.enabled + ) + + def __update_root_depth(self) -> None: + """Restrict the roots with the water table of the previous step. + + The effective root depth is the depth of the water table below the + terrain, bounded by the minimum root depth (``Dpz_min``) and the + rootzone depth (``Zr``) [cm]. Without a previous head (the first step) + or where it is missing, the roots keep the whole rootzone. + """ + rootzone_depth = self.soil_rootzone_depth + if self.previous_water_table_head is None: + self.groundwater_depth_cm = None + self.effective_root_depth = rootzone_depth + else: + self.groundwater_depth_cm = pcr.max( + (self.surface_elevation - self.previous_water_table_head) * 100.0, 0.0 + ) + self.effective_root_depth = pcr.min( + rootzone_depth, + pcr.max( + self.soil_rootzone_depth_min, + pcr.cover(self.groundwater_depth_cm, rootzone_depth), + ), + ) + self.root_depth_fraction = self.effective_root_depth / rootzone_depth + + def __finish_modflow(self) -> None: + """Release the MODFLOW model and remove its run directory after the last step. + + A step that raises never gets here, so a failed run keeps the + directory (and ``pcrmf.lst``) for inspection. + """ + self.modflow = None + shutil.rmtree(self.modflow_run_dir) + self.logger.debug("Removed the MODFLOW run directory %s.", self.modflow_run_dir) def __release_step_state(self): """Drop the fluxes of the step that has just been reported. @@ -559,8 +699,11 @@ def __release_step_state(self): self.current_cell_total_discharge = None self.accumulated_cell_total_discharge = None self.current_runoff = None + self.groundwater_depth_cm = None + self.effective_root_depth = None + self.root_depth_fraction = None - def __current_step_report(self): + def __current_step_report(self, modflow_result: "ModflowStepResult | None" = None): output_vars_dict = { self.config.output_variables.itp.id: self.current_interception, self.config.output_variables.bfw.id: self.current_baseflow, @@ -574,28 +717,66 @@ def __current_step_report(self): } self.__report_raster_series(output_vars_dict) + if modflow_result is not None: + self.__report_modflow_rasters(modflow_result) if self.__time_series_requested(): self.__report_time_series(output_vars_dict) def __report_raster_series(self, output_vars_dict): for var in self.config.output_variables.get_enabled_raster_series(): - if OutputFileFormat.PCRASTER in self.config.output_variables.file_formats: - self.report( - variable=output_vars_dict.get(var.id), - name=str(Path(self.config.output_directory.path) / var.raster_filename_prefix), - ) + self.__report_raster(output_vars_dict.get(var.id), var.raster_filename_prefix) + + def __report_modflow_rasters(self, result: "ModflowStepResult") -> None: + """Write the MODFLOW diagnostics the section enables, in the run's raster formats. + + Prefixes: ``mfh`` head of layer ``n`` [m]; ``mfaq2rv``, ``mfrv2aq`` + and ``mfrvnet`` aquifer-to-river, river-to-aquifer and net (positive + into the aquifer) river leakage [m3/day]; ``mfst`` storage flow and + ``mfdrn`` drain flow of layer ``n`` [m3/day]; with the root-depth + coupling, ``mfwt`` water-table head of the step [m], ``mfgwd`` depth of + the previous step's water table [cm] (missing on the first step), + ``mfzr`` effective root depth [cm] and ``mfzfrac`` its fraction of the + rootzone depth [-]. + """ + output = self.config.modflow.output + rasters = {f"mfh{number}": head for number, head in result.heads.items()} + if output.river_leakage: + rasters["mfaq2rv"] = result.aquifer_to_river_m3_per_day + rasters["mfrv2aq"] = result.river_to_aquifer_m3_per_day + rasters["mfrvnet"] = result.net_river_leakage_m3_per_day + rasters.update({f"mfst{number}": flow for number, flow in result.storage.items()}) + rasters.update({f"mfdrn{number}": flow for number, flow in result.drain_flow.items()}) + if output.root_depth and self.root_depth_fraction is not None: + rasters["mfwt"] = result.water_table_head + rasters["mfgwd"] = ( + pcr.ifthen(pcr.spatial(pcr.boolean(0)), pcr.scalar(0.0)) + if self.groundwater_depth_cm is None + else self.groundwater_depth_cm + ) + rasters["mfzr"] = self.effective_root_depth + rasters["mfzfrac"] = self.root_depth_fraction + for prefix, variable in rasters.items(): + self.__report_raster(variable, prefix) + + def __report_raster(self, variable: Field, prefix: str) -> None: + """Write ``variable`` for the current step in every raster format of the run.""" + if OutputFileFormat.PCRASTER in self.config.output_variables.file_formats: + self.report( + variable=variable, + name=str(Path(self.config.output_directory.path) / prefix), + ) - if OutputFileFormat.GEOTIFF in self.config.output_variables.file_formats: - report( - variable=output_vars_dict.get(var.id), - name=var.raster_filename_prefix, - timestep=self.currentStep, - outpath=self.config.output_directory.path, - file_format=OutputFileFormat.GEOTIFF, - base_raster_info=self.config.output_raster_base, - no_data_value=self.config.output_variables.no_data_value, - ) + if OutputFileFormat.GEOTIFF in self.config.output_variables.file_formats: + report( + variable=variable, + name=prefix, + timestep=self.currentStep, + outpath=self.config.output_directory.path, + file_format=OutputFileFormat.GEOTIFF, + base_raster_info=self.config.output_raster_base, + no_data_value=self.config.output_variables.no_data_value, + ) def __report_time_series(self, output_vars_dict): for var in self.config.output_variables.get_enabled_time_series(): diff --git a/rubem/calibration/_worker.py b/rubem/calibration/_worker.py index b0d1e8b..312a4a2 100644 --- a/rubem/calibration/_worker.py +++ b/rubem/calibration/_worker.py @@ -39,6 +39,7 @@ import numpy as np from ..configuration.model_configuration_file_v1 import VARIABLE_IDS +from .modflow_parameters import ModflowCatalog from .objective import ( INADMISSIBLE_OBJECTIVE, Series, @@ -100,6 +101,11 @@ class EvaluationContext: default, averages every station the two series share; the others are measured either way. :type stations: tuple[str, ...] | None + + :param modflow: The MODFLOW parameters of the configuration, which write the + MODFLOW values of a candidate into its document. ``None``, the default, + when the configuration does not enable MODFLOW. + :type modflow: rubem.calibration.modflow_parameters.ModflowCatalog | None """ document: dict @@ -112,6 +118,7 @@ class EvaluationContext: validate_input: bool = False space: DecisionSpace = field(default_factory=decision_space) stations: tuple[str, ...] | None = None + modflow: ModflowCatalog | None = None def evaluate(vector: Sequence[float] | np.ndarray, context: EvaluationContext) -> float: @@ -202,6 +209,7 @@ def simulate_best( parameters: dict[str, float], output_dir: str, variable: str, + modflow: ModflowCatalog | None = None, ) -> str: """Run one candidate and keep what it wrote. @@ -222,7 +230,8 @@ def simulate_best( on, ``None`` when they are absolute already. :type base_dir: str | None - :param parameters: The nine calibration parameters of the candidate. + :param parameters: The nine calibration parameters of the candidate, and + its MODFLOW parameters when the calibration has any. :type parameters: dict[str, float] :param output_dir: Directory the run writes into. It is not removed. @@ -231,6 +240,10 @@ def simulate_best( :param variable: Id of the output variable the table is wanted for. :type variable: str + :param modflow: The MODFLOW parameters of the configuration, ``None``, the + default, when it does not enable MODFLOW. + :type modflow: rubem.calibration.modflow_parameters.ModflowCatalog | None + :return: The path of the CSV table the run wrote for ``variable``. :rtype: str @@ -238,7 +251,7 @@ def simulate_best( """ from ..api import Model - derived = _document_for(document, variable, parameters, output_dir) + derived = _document_for(document, variable, parameters, output_dir, modflow=modflow) result = Model.from_config(derived, validate_input=False, base_dir=base_dir).run() return str(_time_series_table(result, variable)) @@ -280,7 +293,9 @@ def _derived_document( output_dir: str, ) -> dict: """Return the configuration document of one candidate of an evaluation.""" - return _document_for(context.document, context.variable, parameters, output_dir) + return _document_for( + context.document, context.variable, parameters, output_dir, modflow=context.modflow + ) def _document_for( @@ -288,6 +303,8 @@ def _document_for( variable: str, parameters: dict[str, float], output_dir: str, + *, + modflow: ModflowCatalog | None = None, ) -> dict: """Return the configuration document of one candidate. @@ -297,7 +314,9 @@ def _document_for( no raster, and writing them would dominate its cost) and the time series are reduced to the calibrated variable, as CSV. The aggregation the user configured is kept, since it decides which areas the observed stations - correspond to. + correspond to. With a MODFLOW catalog, the MODFLOW values of the candidate + replace the ones of its section, and the conductivity tables it rewrites + are written into ``output_dir``, which is removed with the run. """ document = copy.deepcopy(base_document) document["model_calibration_parameters"] = { @@ -325,6 +344,8 @@ def _document_for( "formats": ["CSV"], "aggregation": previous_samples.get("aggregation", "point"), } + if modflow is not None: + modflow.apply(document, parameters, output_dir) return document diff --git a/rubem/calibration/modflow_parameters.py b/rubem/calibration/modflow_parameters.py new file mode 100644 index 0000000..2a5bc34 --- /dev/null +++ b/rubem/calibration/modflow_parameters.py @@ -0,0 +1,231 @@ +"""The MODFLOW parameters a calibration may search, and how a candidate reaches the run. + +A configuration that enables MODFLOW offers a few more numbers to calibrate than +the nine of the model. They are named after where they live in the ``modflow`` +section, with the layers numbered from the top down as in the configuration: + +``modflow.layers..specific_yield``, ``modflow.layers..specific_storage`` + The storage of layer ``n`` [-], when the section gives it as a number, the + run is transient and the LAYCON of the layer reads it (LAYCON 0 reads the + specific storage, LAYCON 1 the specific yield, LAYCON 2 and 3 both). +``modflow.layers..kh.`` + The horizontal conductivity of one class of the lookup table of layer + ``n`` [m/day]; a numeric class key is spelt canonically (``01`` and + ``1.0`` are class ``1``). Only the row PCRaster reads for a class is named: + an interval row is not, nor a row whose class an earlier row, numeric or + interval, already matches. +``modflow.river..conductance`` + The numeric conductance of river entry ``i`` [m2/day], counted from 1. + +A map path is not a number to search, so it has no name. The parameters enter a +search only when ``--bound`` or ``--fix`` names them; see +:func:`rubem.calibration.parameters.decision_space`. + +:class:`ModflowCatalog` is plain data, since it travels to the spawned worker +processes, and this module imports no native library. +""" + +import math +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from .._paths import PathInput, as_path +from ..configuration.modflow_configuration import ConductivityLookup, ModflowSettings +from ..validation.lookup_tables import ( + _NUMBER, + _canonical, + _first_matching_row, + read_lookup_table, +) + +MODFLOW_PREFIX = "modflow." +"""The prefix of every MODFLOW parameter name.""" + +_STORAGE_READ = { + 0: ("specific_storage",), + 1: ("specific_yield",), + 2: ("specific_yield", "specific_storage"), + 3: ("specific_yield", "specific_storage"), +} +"""The storage keys the run reads for each LAYCON of a transient run.""" + + +@dataclass(frozen=True) +class ModflowCatalog: + """The MODFLOW parameters of one configuration and their current values. + + :param values: The current value of every calibratable parameter, by name, + in the order of the section: layer by layer from the top, then the + river entries. + :type values: dict[str, float] + + :param tables: The rows of the conductivity lookup table of each layer that + has one, by layer number, as ``(key, value)`` pairs with the keys kept + as they are written. + :type tables: dict[int, list[tuple[str, float]]] + """ + + values: dict[str, float] + tables: dict[int, list[tuple[str, float]]] + + @property + def names(self) -> tuple[str, ...]: + """The calibratable names, in the order of :attr:`values`. + + :return: The names. + :rtype: tuple[str, ...] + """ + return tuple(self.values) + + def domain(self, name: str) -> str: + """Describe the values a parameter may take, for the messages. + + :param name: A name of the catalog. + :type name: str + + :return: ``(0, 1]`` for a specific yield, ``(0, inf)`` otherwise. + :rtype: str + """ + return "(0, 1]" if name.endswith(".specific_yield") else "(0, inf)" + + def admits(self, name: str, value: float) -> bool: + """Whether a parameter may take a value. + + Every MODFLOW parameter is strictly positive and finite: a zero + conductivity or storage would only surface as a run that does not + converge. A specific yield is a fraction, at most 1. + + :param name: A name of the catalog. + :type name: str + + :param value: The candidate value. + :type value: float + + :return: ``True`` when the value lies in the domain of the parameter. + :rtype: bool + """ + value = float(value) + if not math.isfinite(value) or value <= 0.0: + return False + return value <= 1.0 or not name.endswith(".specific_yield") + + def apply( + self, + document: dict, + parameters: Mapping[str, float], + table_dir: PathInput, + *, + table_prefix: str = "kh_layer", + ) -> None: + """Write the MODFLOW values of a candidate into a format 1.0 document. + + The numbers replace the ones of the ``modflow`` section in place. The + classes named for a layer are written, with the other rows of its + table unchanged, into ``/.tbl``, and the + lookup of the layer is pointed at that file; the configured table is + never modified. Names outside the ``modflow.`` prefix are ignored, so + the whole parameter set of a candidate can be given. + + :param document: The configuration document, patched in place. + :type document: dict + + :param parameters: The parameters of the candidate, by name. + :type parameters: collections.abc.Mapping[str, float] + + :param table_dir: Directory the rewritten tables are written to; it + must exist when a class is named. + :type table_dir: str | os.PathLike[str] + + :param table_prefix: File name of a rewritten table, before the layer + number. Defaults to ``kh_layer``. + :type table_prefix: str, optional + + :raises ValueError: If a ``modflow.`` name is not in the catalog. + """ + named = { + name: float(value) + for name, value in parameters.items() + if name.startswith(MODFLOW_PREFIX) + } + unknown = [name for name in named if name not in self.values] + if unknown: + raise ValueError( + f"{', '.join(unknown)} is not a MODFLOW parameter of this configuration; " + f"the calibratable ones are {', '.join(self.names) or 'none'}." + ) + if not named: + return + + section = document["modflow"] + classes: dict[int, dict[str, float]] = {} + for name, value in named.items(): + parts = name.split(".") + if parts[1] == "river": + section["river"]["entries"][int(parts[2]) - 1]["conductance"] = value + elif parts[3] == "kh": + classes.setdefault(int(parts[2]), {})[".".join(parts[4:])] = value + else: + section["layers"][int(parts[2]) - 1][parts[3]] = value + + for number, replaced in classes.items(): + path = as_path(table_dir) / f"{table_prefix}{number}.tbl" + _write_table(path, self.tables[number], replaced) + section["layers"][number - 1]["horizontal_conductivity"]["table"] = str(path) + + +def catalog(settings: ModflowSettings) -> ModflowCatalog: + """Return the MODFLOW parameters a configuration offers to a calibration. + + :param settings: The MODFLOW section, with its paths anchored. + :type settings: rubem.configuration.modflow_configuration.ModflowSettings + + :return: The calibratable names with their current values, and the rows of + every conductivity lookup table. + :rtype: ModflowCatalog + + :raises rubem.validation.lookup_tables.LookupTableError: If a conductivity + lookup table cannot be parsed. + :raises OSError: If a conductivity lookup table cannot be read. + """ + values: dict[str, float] = {} + tables: dict[int, list[tuple[str, float]]] = {} + transient = not settings.dis.steady_state + for number, layer in enumerate(settings.layers, start=1): + prefix = f"{MODFLOW_PREFIX}layers.{number}" + for key in _STORAGE_READ[layer.laycon] if transient else (): + value = getattr(layer, key) + if isinstance(value, float | int): + values[f"{prefix}.{key}"] = float(value) + if isinstance(layer.horizontal_conductivity, ConductivityLookup): + keyed = read_lookup_table(layer.horizontal_conductivity.table) + rows = [(keys[0], value) for keys, value in keyed] + tables[number] = rows + for index, (key, value) in enumerate(rows): + # PCRaster reads the first row that matches a class, numeric or + # interval: a later row of the class never reaches the run. + if _NUMBER.match(key) and _first_matching_row(keyed, float(key)) == index: + values[f"{prefix}.kh.{_canonical(key)}"] = value + for index, entry in enumerate(settings.river.entries, start=1): + if isinstance(entry.conductance, float | int): + values[f"{MODFLOW_PREFIX}river.{index}.conductance"] = float(entry.conductance) + return ModflowCatalog(values=values, tables=tables) + + +def _write_table(path: Path, rows: list[tuple[str, float]], replaced: Mapping[str, float]) -> None: + """Write a lookup table, the first row of each named class carrying its new value.""" + lines = [] + pending = dict(replaced) + for key, value in rows: + canonical = _canonical(key) if _NUMBER.match(key) else None + if canonical in pending: + value = pending.pop(canonical) + lines.append(f"{key} {_number(value)}\n") + path.write_text("".join(lines), encoding="utf8") + + +def _number(value: float) -> str: + """Spell a value in positional notation, the shortest that reads back exactly.""" + return np.format_float_positional(float(value), trim="-") diff --git a/rubem/calibration/parameters.py b/rubem/calibration/parameters.py index 78c16ef..9fd022a 100644 --- a/rubem/calibration/parameters.py +++ b/rubem/calibration/parameters.py @@ -22,6 +22,14 @@ configuration the model accepts. An override may only narrow them, never widen them, for the same reason. +A configuration that enables MODFLOW adds the parameters of its ``modflow`` +section, named in :mod:`rubem.calibration.modflow_parameters`. They are never +searched by default: a MODFLOW parameter joins the vector only when the caller +bounds it, after the eight parameters above, or fixes it, and its bound is +checked against the values the parameter may take instead of a range of the +application settings. Without MODFLOW names the space is the one described +above. + The module-level :func:`bounds`, :func:`vector_to_parameters`, :func:`parameters_to_vector`, :func:`is_admissible` and :func:`weights_constraint` are the default space, the one without fixed @@ -41,6 +49,7 @@ import numpy as np from ..configuration._ranges import variable_range +from .modflow_parameters import MODFLOW_PREFIX, ModflowCatalog if TYPE_CHECKING: from scipy.optimize import LinearConstraint @@ -79,7 +88,8 @@ class DecisionSpace: evaluation context, so they hold nothing but names and numbers. :param free_names: The searched parameters, in the order of the decision - vector: the entries of :data:`FREE_PARAMETERS` that are not fixed. + vector: the entries of :data:`FREE_PARAMETERS` that are not fixed, then + the bounded MODFLOW parameters. :type free_names: tuple[str, ...] :param fixed: The parameters that are not searched, by name, with the value @@ -89,11 +99,17 @@ class DecisionSpace: :param bounds: One ``(minimum, maximum)`` pair per free parameter, in the order of :attr:`free_names`. :type bounds: tuple[tuple[float, float], ...] + + :param modflow_names: The MODFLOW parameters of the run, searched or + fixed, in the order of the MODFLOW catalog. Empty, the default, when + the run calibrates none. + :type modflow_names: tuple[str, ...] """ free_names: tuple[str, ...] fixed: dict[str, float] bounds: tuple[tuple[float, float], ...] + modflow_names: tuple[str, ...] = () @property def dimension(self) -> int: @@ -110,13 +126,14 @@ def to_parameters(self, vector: Sequence[float] | np.ndarray) -> dict[str, float The free values are read in the order of :attr:`free_names`, the fixed ones are inserted at the value they were pinned to, and the slope factor weight is derived as ``w_3 = 1 - w_1 - w_2``, so that the three weights - always add up to 1. + always add up to 1. The MODFLOW parameters of the run follow the nine. :param vector: The free values, in the order of :attr:`free_names`. :type vector: collections.abc.Sequence[float] | numpy.ndarray :return: The nine parameters, keyed by the names of - :class:`rubem.configuration.calibration_parameters.CalibrationParameters`. + :class:`rubem.configuration.calibration_parameters.CalibrationParameters`, + then the :attr:`modflow_names`. :rtype: dict[str, float] :raises ValueError: If the vector does not have :attr:`dimension` entries. @@ -139,7 +156,7 @@ def to_parameters(self, vector: Sequence[float] | np.ndarray) -> dict[str, float # then refuses the run. With a single subtraction the weight is exactly # zero at the boundary and never negative while ``w_1 + w_2 <= 1``. parameters[DERIVED_PARAMETER] = 1.0 - (parameters["w_1"] + parameters["w_2"]) - return {name: parameters[name] for name in CALIBRATION_PARAMETERS} + return {name: parameters[name] for name in (*CALIBRATION_PARAMETERS, *self.modflow_names)} def from_parameters(self, parameters: Mapping[str, float]) -> np.ndarray: """Return the decision vector of a set of calibration parameters. @@ -240,6 +257,7 @@ def weights_constraint(self) -> "LinearConstraint | None": def decision_space( fixed: Mapping[str, float] | None = None, bounds: Mapping[str, tuple[float, float]] | None = None, + modflow: ModflowCatalog | None = None, ) -> DecisionSpace: """Return the decision space of one calibration. @@ -259,6 +277,13 @@ def decision_space( decision vector. Fixing both of them derives ``w_3`` once and checks it here, before the search starts. + A name that starts with ``modflow.`` is a parameter of the MODFLOW section, + looked up in ``modflow``. It has no range in the application settings, so + it is searched only when it is bounded, after the eight parameters, and the + bound is mandatory: finite, its minimum below its maximum and inside the + values the parameter may take (strictly positive, and at most 1 for a + specific yield). A fixed one keeps its value in every candidate. + :param fixed: The parameters that are not searched, with the value every candidate carries. Defaults to ``None``, no fixed parameter. :type fixed: collections.abc.Mapping[str, float], optional @@ -267,6 +292,10 @@ def decision_space( by name. Defaults to ``None``, the ranges of the application settings. :type bounds: collections.abc.Mapping[str, tuple[float, float]], optional + :param modflow: The MODFLOW parameters of the configuration, ``None``, the + default, when it does not enable MODFLOW. + :type modflow: rubem.calibration.modflow_parameters.ModflowCatalog, optional + :return: The decision vector of the run. :rtype: DecisionSpace @@ -274,13 +303,29 @@ def decision_space( ``w_3``, if a fixed value lies outside the range of the settings, if an overridden bound is not a narrower range of the settings range, if a fixed weight leaves the other one no admissible value, if two fixed - weights derive a ``w_3`` outside its range, or if every parameter is - fixed and nothing is left to search. + weights derive a ``w_3`` outside its range, if every parameter is fixed + and nothing is left to search, or if a MODFLOW name is given without + ``modflow``, is not in it, or has a bound or a fixed value outside the + values it may take. """ + fixed = fixed or {} + bounds = bounds or {} + modflow_space = _modflow_space( + {name: value for name, value in fixed.items() if name.startswith(MODFLOW_PREFIX)}, + {name: value for name, value in bounds.items() if name.startswith(MODFLOW_PREFIX)}, + modflow, + ) + modflow_free, modflow_fixed, modflow_bounds, modflow_names = modflow_space fixed_values = { - _canonical(name, "fixed"): float(value) for name, value in (fixed or {}).items() + _canonical(name, "fixed"): float(value) + for name, value in fixed.items() + if not name.startswith(MODFLOW_PREFIX) + } + overrides = { + _canonical(name, "bounded"): value + for name, value in bounds.items() + if not name.startswith(MODFLOW_PREFIX) } - overrides = {_canonical(name, "bounded"): value for name, value in (bounds or {}).items()} settings_ranges = {name: variable_range(name) for name in FREE_PARAMETERS} effective = dict(settings_ranges) @@ -301,7 +346,7 @@ def decision_space( ) free_names = tuple(name for name in FREE_PARAMETERS if name not in fixed_values) - if not free_names: + if not free_names and not modflow_free: raise ValueError( "Every calibration parameter is fixed, so the search has nothing to look for; " "leave at least one of them free." @@ -319,15 +364,77 @@ def decision_space( if minimum == maximum: fixed_values[name] = minimum free_names = tuple(free for free in free_names if free != name) - if not free_names: + if not free_names and not modflow_free: raise ValueError( "Every calibration parameter is fixed, so the search has nothing to look for; " "leave at least one of them free." ) return DecisionSpace( - free_names=free_names, - fixed=fixed_values, - bounds=tuple(effective[name] for name in free_names), + free_names=(*free_names, *modflow_free), + fixed={**fixed_values, **modflow_fixed}, + bounds=(*(effective[name] for name in free_names), *modflow_bounds), + modflow_names=modflow_names, + ) + + +def _modflow_space( + fixed: Mapping[str, float], + bounds: Mapping[str, Sequence[float]], + modflow: ModflowCatalog | None, +) -> tuple[tuple[str, ...], dict[str, float], tuple[tuple[float, float], ...], tuple[str, ...]]: + """Return the searched and the fixed MODFLOW parameters, in the order of the catalog. + + :return: The searched names, the fixed values, the bounds of the searched + names and every MODFLOW name of the run. + + :raises ValueError: If a name is given without a catalog or is not in it, + if it is both fixed and bounded, or if its value or its bound lies + outside the values the parameter may take. + """ + given = [*fixed, *bounds] + if not given: + return (), {}, (), () + if modflow is None: + raise ValueError( + f"The MODFLOW parameter(s) {', '.join(given)} cannot be calibrated: the " + "configuration does not enable MODFLOW." + ) + for name in given: + if name not in modflow.values: + raise ValueError( + f"'{name}' is not a MODFLOW parameter of this configuration. Its calibratable " + f"MODFLOW parameters are {', '.join(modflow.names) or 'none'}." + ) + for name in bounds: + if name in fixed: + raise ValueError( + f"'{name}' is fixed at {fixed[name]}, so a bound for it has nowhere " + "to apply: only the free parameters are searched." + ) + for name, value in fixed.items(): + if not modflow.admits(name, value): + raise ValueError( + f"'{name}' cannot be fixed at {value}: the value lies outside " + f"{modflow.domain(name)}, the values the parameter may take." + ) + checked = {} + for name, override in bounds.items(): + minimum, maximum = _pair(name, override) + if not ( + minimum < maximum and modflow.admits(name, minimum) and modflow.admits(name, maximum) + ): + raise ValueError( + f"The bound ({minimum}, {maximum}) of '{name}' is not a range inside " + f"{modflow.domain(name)}, the values the parameter may take, with its minimum " + "below its maximum." + ) + checked[name] = (minimum, maximum) + searched = tuple(name for name in modflow.names if name in checked) + return ( + searched, + {name: float(fixed[name]) for name in modflow.names if name in fixed}, + tuple(checked[name] for name in searched), + tuple(name for name in modflow.names if name in fixed or name in checked), ) @@ -360,19 +467,7 @@ def _narrowed( :raises ValueError: If the override is not a range with a minimum below its maximum, or if it is not contained in the range of the settings. """ - try: - values = tuple(float(value) for value in override) - except TypeError: - # A single number instead of a pair: the same mistake as a pair of the - # wrong length, and it is answered with the same sentence. - raise ValueError( - f"The bound of '{name}' must be a (minimum, maximum) pair, got {override!r}." - ) from None - if len(values) != 2: - raise ValueError( - f"The bound of '{name}' must be a (minimum, maximum) pair, got {len(values)} value(s)." - ) - minimum, maximum = values + minimum, maximum = _pair(name, override) settings_minimum, settings_maximum = settings_range if ( not math.isfinite(minimum) @@ -389,6 +484,27 @@ def _narrowed( return minimum, maximum +def _pair(name: str, override: Sequence[float]) -> tuple[float, float]: + """Return a bound as a pair of numbers. + + :raises ValueError: If the bound is not two numbers. + """ + try: + values = tuple(float(value) for value in override) + except TypeError: + # A single number instead of a pair: the same mistake as a pair of the + # wrong length, and it is answered with the same sentence. + raise ValueError( + f"The bound of '{name}' must be a (minimum, maximum) pair, got {override!r}." + ) from None + if len(values) != 2: + raise ValueError( + f"The bound of '{name}' must be a (minimum, maximum) pair, got {len(values)} value(s)." + ) + minimum, maximum = values + return minimum, maximum + + def _weight_bounds( fixed_values: Mapping[str, float], effective: Mapping[str, tuple[float, float]] ) -> dict[str, tuple[float, float]]: diff --git a/rubem/calibration/runner.py b/rubem/calibration/runner.py index e7610e5..9367cd5 100644 --- a/rubem/calibration/runner.py +++ b/rubem/calibration/runner.py @@ -61,6 +61,7 @@ import numpy as np +from .. import _deps from .._paths import PathInput, as_path from ..configuration._ranges import variable_range from ..configuration.model_configuration_file_v1 import ( @@ -68,7 +69,9 @@ Aggregation, ModelConfigurationFileV1, ) +from ..validation.lookup_tables import LookupTableError from ._worker import _INADMISSIBLE_ERROR, EvaluationContext, evaluate, simulate_best +from .modflow_parameters import MODFLOW_PREFIX, ModflowCatalog, catalog from .objective import Series, align_series, read_series, valid_mask from .parameters import ( CALIBRATION_PARAMETERS, @@ -106,7 +109,8 @@ "elapsed_seconds", "error", ) -"""The columns of ``evaluations.csv``, one row per evaluation.""" +"""The columns of ``evaluations.csv``, one row per evaluation, when the run +calibrates no MODFLOW parameter; the MODFLOW parameters of a run follow ``x``.""" OBSERVED_COLUMNS = ( "station", @@ -138,6 +142,14 @@ """The separator of the per-station tables, the one the model writes its own tables with.""" +_MODFLOW_DEATH = ( + " With MODFLOW, a run that does not converge before the last time step of a stress " + "period (dis.nstp > 1) also ends its worker process; dis.nstp = 1 turns it into a " + "failed evaluation." +) +"""What a dead worker of a coupled calibration may also mean.""" + + class CalibrationError(RuntimeError): """A calibration that could not start, or that produced no usable evaluation.""" @@ -250,7 +262,8 @@ class CalibrationResult: """What a finished calibration found and where it wrote it. :param best_parameters: The nine calibration parameters of the best - candidate, the derived ``w_3`` included. + candidate, the derived ``w_3`` included, followed by the MODFLOW + parameters the run searched or fixed. :type best_parameters: dict[str, float] :param best_nse: The mean Nash-Sutcliffe efficiency of the best candidate, @@ -404,15 +417,17 @@ def calibrate( :raises CalibrationError: If SciPy is not installed, if the configuration cannot be calibrated as asked, if the observed series shares no time step or no station with what the configuration will sample, if the - station selection names none of those stations, if a worker process + station selection names none of those stations, if a conductivity + lookup table of the MODFLOW section cannot be read, if a worker process died before finishing its evaluation, or if every evaluation of the run failed. An interrupted or crashed search still leaves the table of the evaluations it made behind. :raises ValueError: If the decision space the settings describe is not a search (an unknown or derived parameter, a fixed value or an overridden bound outside the range of the application settings, every parameter - fixed), or if an explicit initial population does not have one column per - free parameter. + fixed, a MODFLOW name the configuration does not offer or a MODFLOW bound + outside the values of its parameter), or if an explicit initial + population does not have one column per free parameter. :raises ImportError: If PCRaster or GDAL are not installed. :raises FileNotFoundError: If the configuration or the observed series is not there. @@ -437,17 +452,6 @@ def calibrate( ) _check_stations(settings.stations) - space = decision_space(fixed=settings.fixed, bounds=settings.bounds) - if not isinstance(settings.init, str): - shape = np.shape(settings.init) - if len(shape) != 2 or shape[1] != space.dimension: - raise ValueError( - f"The initial population has the shape {shape} and the search has " - f"{space.dimension} free parameter(s) ({', '.join(space.free_names)}); it must " - f"be a two-dimensional array of {shape[0] if shape else 0} member(s) with one " - "column per free parameter." - ) - config_file = as_path(config_path).absolute() # The only validation of a whole calibration: the workers never revalidate, # so this is where a blocking problem stops the search, and where @@ -458,6 +462,29 @@ def calibrate( allow_blocking_problems=settings.allow_blocking_problems, ).configuration file_v1 = _as_v1(configuration) + modflow = _modflow_catalog(configuration) + if modflow is not None and configuration.modflow.dis.nstp > 1: + # The extension ends the process, not the stress period, when MODFLOW + # stops before the last time step; the pool cannot replace that worker. + logger.warning( + "The MODFLOW section has dis.nstp %d: a candidate whose solver does not " + "converge before the last time step of a stress period ends its worker " + "process, and with it the calibration, instead of being ranked last as a " + "failed evaluation. Set dis.nstp to 1 to keep the search running past it.", + configuration.modflow.dis.nstp, + ) + # The MODFLOW names can only be resolved once the configuration says which + # parameters its MODFLOW section offers. + space = decision_space(fixed=settings.fixed, bounds=settings.bounds, modflow=modflow) + if not isinstance(settings.init, str): + shape = np.shape(settings.init) + if len(shape) != 2 or shape[1] != space.dimension: + raise ValueError( + f"The initial population has the shape {shape} and the search has " + f"{space.dimension} free parameter(s) ({', '.join(space.free_names)}); it must " + f"be a two-dimensional array of {shape[0] if shape else 0} member(s) with one " + "column per free parameter." + ) _check_sample_locations(configuration, file_v1) observed = read_series(observed_path) @@ -481,7 +508,13 @@ def calibrate( station_ids=sampled_ids, ) _check_station_selection(settings.stations, observed, sampled_ids) - x0 = _starting_point(space, configuration.calibration_parameters.model_dump()) + x0 = _starting_point( + space, + { + **configuration.calibration_parameters.model_dump(), + **(modflow.values if modflow is not None else {}), + }, + ) run_directory = as_path(run_dir).absolute() evaluations_dir = run_directory / EVALUATIONS_DIRNAME @@ -517,7 +550,9 @@ def calibrate( validate_input=False, space=space, stations=settings.stations, + modflow=modflow, ) + columns = _evaluation_columns(space) planned_members = _population_size(settings, space.dimension) progress.info( @@ -537,6 +572,10 @@ def calibrate( # out instead of being passed as ``None``. constraint = space.weights_constraint() callback = _Progress(evaluations_dir) + if modflow is not None: + # The extension launches mf2005 from PATH, and the spawned workers + # inherit the environment of this process. + _deps.ensure_mf2005_on_path() executor = _pool(workers) arguments: dict[str, Any] = { "strategy": settings.strategy, @@ -568,18 +607,19 @@ def calibrate( # An interruption or a crash after the first evaluation still leaves a # table: the evaluations that were made are hours of model runs, and a # calibration that has to be stopped is read from them like any other. - _consolidate(evaluations_csv, evaluations_dir) + _consolidate(evaluations_csv, evaluations_dir, columns) if isinstance(error, BrokenProcessPool): raise CalibrationError( "A worker process died before finishing its evaluation, most often " "because the system killed it for lack of memory: every worker holds " "the rasters of one model run. Run the calibration again with a lower " - f"--workers. The evaluations made so far are in {evaluations_csv}." + f"--workers.{_MODFLOW_DEATH if modflow is not None else ''} The evaluations " + f"made so far are in {evaluations_csv}." ) from error raise records = _read_records(evaluations_dir) - _write_evaluations_csv(evaluations_csv, records) + _write_evaluations_csv(evaluations_csv, records, columns) _report_failures(records) best = _best_record(records) @@ -608,12 +648,13 @@ def calibrate( stations_csv = run_directory / STATIONS_CSV best_series = run_directory / f"best_{settings.variable}.csv" _write_stations_csv(stations_csv, best.get("station_metrics") or {}, settings.stations) - _write_calibrated_config(calibrated_config, configuration, file_v1, best_parameters) + _write_calibrated_config(calibrated_config, configuration, file_v1, best_parameters, modflow) _write_best_series( best_series, document=file_v1.to_dict(), base_dir=configuration.base_dir, parameters=best_parameters, + modflow=modflow, variable=settings.variable, observed=observed, spinup_steps=settings.spinup_steps, @@ -727,7 +768,7 @@ def _pool(workers: int) -> ProcessPoolExecutor: ) -def _consolidate(evaluations_csv: Path, evaluations_dir: Path) -> None: +def _consolidate(evaluations_csv: Path, evaluations_dir: Path, columns: tuple[str, ...]) -> None: """Write the table of the evaluations recorded so far, if there are any. This is what an interrupted or crashed search leaves behind: the records @@ -739,7 +780,7 @@ def _consolidate(evaluations_csv: Path, evaluations_dir: Path) -> None: try: records = _read_records(evaluations_dir) if records: - _write_evaluations_csv(evaluations_csv, records) + _write_evaluations_csv(evaluations_csv, records, columns) logger.warning( "The search ended early; the %d evaluation(s) it recorded are in %s.", len(records), @@ -852,10 +893,12 @@ def _report_decision_space(space: DecisionSpace, stations: tuple[str, ...] | Non "Fixed, not searched: %s.", ", ".join(f"{name} = {value:g}" for name, value in space.fixed.items()), ) + # A MODFLOW parameter has no range in the application settings: its bound + # is always the caller's. narrowed = [ (name, bound) for name, bound in zip(space.free_names, space.bounds, strict=True) - if bound != variable_range(name) + if name.startswith(MODFLOW_PREFIX) or bound != variable_range(name) ] if narrowed: logger.info( @@ -1123,6 +1166,7 @@ def _write_best_series( document: dict, base_dir: str | None, parameters: dict[str, float], + modflow: ModflowCatalog | None, variable: str, observed: Series, spinup_steps: int, @@ -1147,7 +1191,7 @@ def _write_best_series( executor = _pool(workers) try: table = executor.submit( - simulate_best, document, base_dir, parameters, str(output_dir), variable + simulate_best, document, base_dir, parameters, str(output_dir), variable, modflow ).result() simulated = read_series(table) except Exception as error: @@ -1240,15 +1284,36 @@ def _record_order(record: dict[str, Any]) -> tuple[tuple[float, ...], str]: identifier only separates two evaluations of the very same candidate. """ parameters = record.get("parameters") or {} - values = tuple(float(parameters.get(name, math.inf)) for name in CALIBRATION_PARAMETERS) + names = ( + *CALIBRATION_PARAMETERS, + *sorted(name for name in parameters if name.startswith(MODFLOW_PREFIX)), + ) + values = tuple(float(parameters.get(name, math.inf)) for name in names) return values, str(record.get("id", "")) -def _write_evaluations_csv(path: Path, records: list[dict[str, Any]]) -> None: - """Write one row per evaluation, in the columns of :data:`EVALUATION_COLUMNS`.""" +def _evaluation_columns(space: DecisionSpace) -> tuple[str, ...]: + """Return the columns of ``evaluations.csv``: the MODFLOW parameters follow ``x``.""" + position = EVALUATION_COLUMNS.index(CALIBRATION_PARAMETERS[-1]) + 1 + return ( + *EVALUATION_COLUMNS[:position], + *space.modflow_names, + *EVALUATION_COLUMNS[position:], + ) + + +def _write_evaluations_csv( + path: Path, records: list[dict[str, Any]], columns: tuple[str, ...] +) -> None: + """Write one row per evaluation, in ``columns``, which :func:`_evaluation_columns` gives. + + The columns are always given: a default of :data:`EVALUATION_COLUMNS` would + silently drop the MODFLOW parameters of a run on a path that forgot them. + """ + names = columns[columns.index(CALIBRATION_PARAMETERS[0]) : columns.index("nse")] with path.open("w", encoding="utf-8", newline="") as table: writer = csv.writer(table) - writer.writerow(EVALUATION_COLUMNS) + writer.writerow(columns) for record in records: parameters = record.get("parameters") or {} writer.writerow( @@ -1256,7 +1321,7 @@ def _write_evaluations_csv(path: Path, records: list[dict[str, Any]]) -> None: record.get("id", ""), record.get("pid", ""), record.get("started_at", ""), - *(_cell(parameters.get(name)) for name in CALIBRATION_PARAMETERS), + *(_cell(parameters.get(name)) for name in names), _cell(record.get("nse")), _cell(record.get("objective")), _cell(record.get("elapsed_seconds")), @@ -1335,7 +1400,8 @@ def _settings_document( the run actually used, not the ``None`` the caller may have left them at, so that the summary describes a calibration that can be repeated: the bounds are the effective ones, one entry per searched parameter, the ranges of the - application settings narrowed by whatever the caller asked for. + application settings narrowed by whatever the caller asked for, and the + caller's own bound for a MODFLOW parameter. """ init = settings.init if isinstance(settings.init, str) else np.asarray(settings.init).tolist() return { @@ -1366,13 +1432,17 @@ def _write_calibrated_config( configuration, file_v1: ModelConfigurationFileV1, best_parameters: dict[str, float], + modflow: ModflowCatalog | None = None, ) -> None: """Write the calibrated configuration in the format the input was written in. The document goes through the configuration models, so what is written is a configuration the loader accepts. Its paths are the ones the loader resolved, which are absolute even when the input file wrote them relative to its own - directory. + directory. The MODFLOW values of the best candidate replace the ones of its + section, and a conductivity table with a calibrated class is written next + to the configuration as ``-kh.tbl``, ```` being the name of + the configuration without its extension. """ calibrated = file_v1.model_copy( update={ @@ -1389,9 +1459,30 @@ def _write_calibrated_config( ) } ) + if modflow is not None: + patched = calibrated.to_dict() + modflow.apply(patched, best_parameters, path.parent, table_prefix=f"{path.stem}-kh") + calibrated = ModelConfigurationFileV1.model_validate(patched) document = ( calibrated.to_dict() if configuration.file_v1 is not None else calibrated.to_legacy().to_dict() ) path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + + +def _modflow_catalog(configuration) -> ModflowCatalog | None: + """Return the MODFLOW parameters of a configuration, ``None`` when it does not enable MODFLOW. + + :raises CalibrationError: If a conductivity lookup table cannot be read, + which the validation reports as blocking and + ``allow_blocking_problems`` may have let through. + """ + if not configuration.modflow_enabled: + return None + try: + return catalog(configuration.modflow) + except (OSError, LookupTableError) as error: + raise CalibrationError( + f"The MODFLOW parameters of the configuration cannot be read: {error}" + ) from error diff --git a/rubem/cli.py b/rubem/cli.py index cf19f25..7e9d992 100644 --- a/rubem/cli.py +++ b/rubem/cli.py @@ -365,11 +365,25 @@ def calibrate( ] = None, bound: Annotated[ list[str] | None, - typer.Option("--bound", help="Narrow the range of one parameter: NAME=MIN:MAX."), + typer.Option( + "--bound", + help=( + "Narrow the range of one parameter: NAME=MIN:MAX. A MODFLOW parameter " + "(modflow.layers..specific_yield, modflow.layers..specific_storage, " + "modflow.layers..kh., modflow.river..conductance) is searched " + "only when bounded; see the calibration page of the documentation." + ), + ), ] = None, fix: Annotated[ list[str] | None, - typer.Option("--fix", help="Keep one parameter out of the search: NAME=VALUE."), + typer.Option( + "--fix", + help=( + "Keep one parameter out of the search: NAME=VALUE; MODFLOW parameters " + "use the names of --bound." + ), + ), ] = None, stations: Annotated[ str | None, diff --git a/rubem/configuration/migrate.py b/rubem/configuration/migrate.py index f139497..bea1a20 100644 --- a/rubem/configuration/migrate.py +++ b/rubem/configuration/migrate.py @@ -10,6 +10,7 @@ from ._json import read_json from .model_configuration_file import ModelConfigurationFile from .model_configuration_file_v1 import ModelConfigurationFileV1 +from .modflow_configuration import transform_paths logger = logging.getLogger(__name__) @@ -103,4 +104,6 @@ def rebase(value): spec["yearly_file_path"] = rebase(spec["yearly_file_path"]) else: spec["dir_path"] = rebase(spec["dir_path"]) + if "modflow" in document: + transform_paths(document["modflow"], rebase) return document diff --git a/rubem/configuration/model_configuration.py b/rubem/configuration/model_configuration.py index ba04b28..97f5028 100644 --- a/rubem/configuration/model_configuration.py +++ b/rubem/configuration/model_configuration.py @@ -37,6 +37,7 @@ from ..configuration.simulation_period import SimulationPeriod from ..validation.grid_cell_size import check_grid_cell_size from ..validation.lookup_tables import check_lookup_tables, check_runoff_coefficient_domain +from ..validation.modflow_inputs import check_modflow_inputs class ModelConfiguration: @@ -135,6 +136,17 @@ def __init__( self.calibration_parameters.w_3, ) ) + if self.modflow_enabled: + # File existence and the MODFLOW runtime are checked whatever + # validate_input says; the content only with it. + self.problems.extend( + check_modflow_inputs( + self.modflow, + self.raster_files, + validate_input, + tables=self.lookuptable_files, + ) + ) self.__check_inconsistencies(allow_blocking_problems) def __parse(self, data: dict, duplicates: list[str]) -> None: @@ -157,6 +169,7 @@ def __parse(self, data: dict, duplicates: list[str]) -> None: def __build_from_legacy(self, validate_input: bool) -> None: self.logger.debug("Loading configuration...") file = self.file + self.modflow = file.modflow self.simulation_period = SimulationPeriod( start=file.sim_time.start, end=file.sim_time.end, @@ -275,6 +288,7 @@ def __build_from_legacy(self, validate_input: bool) -> None: def __build_from_v1(self, validate_input: bool) -> None: self.logger.debug("Loading configuration (format 1.0)...") file = self.file_v1 + self.modflow = file.modflow period = file.simulation_period self.simulation_period = SimulationPeriod( start=period.start, end=period.finish, alignment=period.alignment @@ -433,6 +447,14 @@ def __check_grid_cell_size(self, validate_input: bool) -> None: if problem is not None: self.problems.append(problem) + @property + def modflow_enabled(self) -> bool: + """Whether the run is coupled to MODFLOW (the section is present and enabled). + + :rtype: bool + """ + return self.modflow is not None and self.modflow.enabled + def write_metadata(self) -> None: """Write ``metadata.json`` next to the outputs of a format 1.0 run. @@ -557,6 +579,18 @@ def __check_inconsistencies(self, allow_blocking_problems: bool): ) ) + if self.modflow is not None and not self.modflow.enabled: + self.problems.append( + Problem( + description="MODFLOW section is ignored.", + reason=( + "The MODFLOW section (modflow in format 1.0) is given but its enabled " + "key is false: the saturated zone is the lumped reservoir of RUBEM and " + "the section is not checked." + ), + ) + ) + if self.problems: self.logger.warning("Configuration problems found: %d", len(self.problems)) for problem in self.problems: diff --git a/rubem/configuration/model_configuration_file.py b/rubem/configuration/model_configuration_file.py index 87559af..8d435d5 100644 --- a/rubem/configuration/model_configuration_file.py +++ b/rubem/configuration/model_configuration_file.py @@ -26,6 +26,7 @@ from .._paths import PathInput, as_path from ._json import read_json +from .modflow_configuration import ModflowSettings logger = logging.getLogger(__name__) @@ -308,6 +309,7 @@ class ModelConfigurationFile(_Section): raster_file_format: RasterFileFormat = Field( alias="RASTER_FILE_FORMAT", default_factory=RasterFileFormat ) + modflow: ModflowSettings | None = Field(alias="MODFLOW", default=None) @classmethod def from_json(cls, path: PathInput) -> Self: @@ -336,8 +338,13 @@ def anchor(value): data["DIRECTORIES"] = directories data["RASTERS"] = rasters data["TABLES"] = tables + if self.modflow is not None: + data["MODFLOW"] = self.modflow.resolve_paths(base) return type(self).model_validate(data) def to_dict(self) -> dict: - """The canonical legacy dictionary (upper-case sections, canonical keys).""" + """The canonical legacy dictionary (upper-case sections, canonical keys). + + An absent optional section is written as ``null`` (``MODFLOW``). + """ return self.model_dump(by_alias=True) diff --git a/rubem/configuration/model_configuration_file_v1.py b/rubem/configuration/model_configuration_file_v1.py index aaf0352..f6f5039 100644 --- a/rubem/configuration/model_configuration_file_v1.py +++ b/rubem/configuration/model_configuration_file_v1.py @@ -33,6 +33,7 @@ from .._paths import PathInput, as_path from ._json import DuplicateKeyWarning from .model_configuration_file import ModelConfigurationFile, finite_float32 +from .modflow_configuration import ModflowSettings VERSION = "1.0" @@ -334,6 +335,7 @@ class ModelConfigurationFileV1(_Strict): model_initial_soil_conditions: InitialSoilConditions model_constants: Constants model_simulation_output: SimulationOutput + modflow: ModflowSettings | None = None @model_validator(mode="after") def _check_aggregation_inputs(self) -> Self: @@ -403,10 +405,12 @@ def anchor(value): spec["yearly_file_path"] = anchor(spec.get("yearly_file_path")) else: spec["dir_path"] = anchor(spec["dir_path"]) + if self.modflow is not None: + data["modflow"] = self.modflow.resolve_paths(base) return type(self).model_validate(data) def to_dict(self) -> dict: - """The JSON document (ISO dates, ``$ref`` objects kept).""" + """The JSON document (ISO dates, ``$ref`` objects kept, unset keys left out).""" return self.model_dump(by_alias=True, mode="json", exclude_none=True) # ----- conversions ----------------------------------------------------- @@ -418,7 +422,8 @@ def from_legacy(cls, legacy: ModelConfigurationFile, metadata: dict | None = Non Directory series become ``dir_path``/``files_prefix`` entries, the period keeps its dates, the legacy time-series flag becomes the ``time_series_samples`` selection (CSV only, as the legacy run writes), - and the raster formats follow ``RASTER_FILE_FORMAT``. + and the raster formats follow ``RASTER_FILE_FORMAT``; the ``MODFLOW`` + section, when present, becomes ``modflow`` unchanged. """ rff = legacy.raster_file_format formats = [] @@ -505,6 +510,8 @@ def from_legacy(cls, legacy: ModelConfigurationFile, metadata: dict | None = Non }, }, } + if legacy.modflow is not None: + data["modflow"] = legacy.modflow.model_dump(mode="json") return cls.model_validate(data) def to_legacy(self) -> ModelConfigurationFile: @@ -595,5 +602,6 @@ def to_legacy(self) -> ModelConfigurationFile: "tiff_raster_series": RasterFormat.GEOTIFF in out.raster_series.formats, "no_data_value": out.raster_series.no_data_value, }, + "MODFLOW": self.modflow.model_dump(mode="json") if self.modflow is not None else None, } return ModelConfigurationFile.model_validate(data) diff --git a/rubem/configuration/modflow_configuration.py b/rubem/configuration/modflow_configuration.py new file mode 100644 index 0000000..df87878 --- /dev/null +++ b/rubem/configuration/modflow_configuration.py @@ -0,0 +1,526 @@ +"""The optional MODFLOW section of the configuration (legacy ``MODFLOW``, format 1.0 ``modflow``). + +The same models are nested in both formats and are strict in both: an +unknown key is refused (a key that no longer exists would otherwise silently +disable a package). Layers are listed from the top down and numbered like +MODFLOW (layer 1 is the top layer); the PCRaster MODFLOW extension numbers +them from the bottom up, and :meth:`ModflowSettings.pcraster_layer` converts. +Lengths are metres and times days, the units of the coupling. + +Cross-field rules are checked only when the section is enabled: a disabled +section is ignored by the run, so it may stay incomplete. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Annotated, Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from .._paths import PathInput, as_path + +MapPath = Annotated[str, Field(min_length=1)] +LayerNumber = Annotated[int, Field(ge=1)] +LayerList = Annotated[list[LayerNumber], Field(min_length=1)] +PositiveInt = Annotated[int, Field(ge=1)] +PositiveFloat = Annotated[float, Field(gt=0, allow_inf_nan=False)] +# ``strict`` keeps a quoted number on the path branch of a ``path | number`` union. +PositiveNumber = Annotated[float, Field(strict=True, gt=0, allow_inf_nan=False)] +NonNegativeNumber = Annotated[float, Field(strict=True, ge=0, allow_inf_nan=False)] +Fraction = Annotated[float, Field(strict=True, ge=0, le=1, allow_inf_nan=False)] + +LAYTYPES = frozenset(tens + laycon for tens in (0, 10, 20, 30) for laycon in (0, 1, 2, 3)) +"""BCF layer types: the tens digit selects the transmissivity averaging, the +units digit (LAYCON) the layer type: 0 confined, 1 unconfined (top layer only), +2 and 3 convertible.""" + +WETTING_LAYCONS = (1, 3) +"""LAYCON values for which BCF reads a WETDRY array.""" + +MAX_LAYERS = 9 +"""Most layers a section may list: the per-layer output prefixes (``mfh``) +are followed by the zero-padded step, so ``mfh1`` and ``mfh10`` would give the +same file name.""" + + +class _Strict(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class ConductivityLookup(_Strict): + """A nominal class map and a PCRaster lookup table giving the conductivity per class. + + :param map: Nominal raster of the conductivity classes. + :param table: PCRaster lookup table, one ``class value`` row per class [m/day]. + """ + + map: MapPath + table: MapPath + + +class ModflowLayer(_Strict): + """One MODFLOW layer: geometry, BAS domain and BCF properties. + + :param name: Unique name used in logs and outputs. + :param bottom: Raster of the bottom elevation of the layer [m]. + :param initial_head: Raster of the starting head [m]. + :param boundary: BAS IBOUND raster: -1 constant head, 0 inactive, 1 active. + :param laytype: BCF layer type (tens digit: averaging; units digit: LAYCON). + :param horizontal_conductivity: Raster, number or class lookup [m/day]. + :param vertical_conductivity: Raster or number [m/day]. + :param specific_storage: Raster or number, the confined storage coefficient + (BCF Sf1) [-]; needed in transient runs by LAYCON 0, 2 and 3. + :param specific_yield: Raster or fraction (BCF Sf2) [-]; needed in + transient runs by LAYCON 1, 2 and 3. + :param compute_conductivity: Whether the extension computes the vertical + conductance from the conductivities. + """ + + name: Annotated[str, Field(min_length=1)] + bottom: MapPath + initial_head: MapPath + boundary: MapPath + laytype: int + horizontal_conductivity: MapPath | PositiveNumber | ConductivityLookup + vertical_conductivity: MapPath | PositiveNumber + specific_storage: MapPath | NonNegativeNumber | None = None + specific_yield: MapPath | Fraction | None = None + compute_conductivity: bool = True + + @field_validator("laytype") + @classmethod + def _known_laytype(cls, value: int) -> int: + if value not in LAYTYPES: + raise ValueError( + f"invalid MODFLOW LAYTYPE {value}: expected one of {sorted(LAYTYPES)}." + ) + return value + + @property + def laycon(self) -> int: + """The LAYCON digit of :attr:`laytype`.""" + return self.laytype % 10 + + +class Dis(_Strict): + """Time discretization of each stress period (one RUBEM step). + + :param nstp: Number of time steps per stress period. One by default: the + extension reads the heads of a period only when its last time step + converged, so a solver failure at an earlier time step ends the whole + process instead of raising. + :param tsmult: Time step multiplier. + :param steady_state: Whether every stress period is steady state. + """ + + nstp: PositiveInt = 1 + tsmult: PositiveFloat = 1.0 + steady_state: bool = False + + +class Solver(_Strict): + """PCG solver settings (the scientist's defaults). + + :param hclose: Head change criterion [m]. + :param rclose: Residual criterion [m3/day]. + """ + + mxiter: PositiveInt = 2000 + iter1: PositiveInt = 20 + npcond: Literal[1, 2] = 1 + hclose: PositiveFloat = 5.0 + rclose: PositiveFloat = 3.0 + relax: PositiveFloat = 1.0 + nbpol: Annotated[int, Field(ge=0)] = 2 + damp: PositiveFloat = 0.5 + + +class Wetting(_Strict): + """BCF rewetting of dry cells. + + :param map: WETDRY raster [m]. + :param layers: Layers the map applies to; ``None`` means every layer + whose LAYCON is 1 or 3. + """ + + enabled: bool = False + map: MapPath | None = None + layers: list[LayerNumber] | None = None + wetfct: PositiveFloat = 1.0 + iwetit: PositiveInt = 3 + ihdwet: Literal[0, 1] = 0 + + +class RiverEntry(_Strict): + """RIV cells of one or more layers. + + :param stage: Raster of the river stage [m]. + :param bottom: Raster of the riverbed bottom [m]. + :param conductance: Raster or number [m2/day]; a number needs ``mask``. + :param mask: Raster whose positive cells are river cells. + """ + + layers: LayerList + stage: MapPath + bottom: MapPath + conductance: MapPath | PositiveNumber + mask: MapPath | None = None + + +class GhbEntry(_Strict): + """GHB cells of one or more layers (the same maps in every listed layer). + + :param head: Raster of the boundary head [m]. + :param conductance: Raster of the boundary conductance [m2/day]. + """ + + layers: LayerList + head: MapPath + conductance: MapPath + + +class DrainEntry(_Strict): + """DRN cells of one or more layers. + + :param elevation: Raster of the drain elevation [m]. + :param conductance: Raster of the drain conductance [m2/day]. + """ + + layers: LayerList + elevation: MapPath + conductance: MapPath + + +class River(_Strict): + enabled: bool = False + entries: list[RiverEntry] = [] + + +class Ghb(_Strict): + enabled: bool = False + entries: list[GhbEntry] = [] + + +class Drain(_Strict): + enabled: bool = False + entries: list[DrainEntry] = [] + + +class WaterTable(_Strict): + """How the water-table head of the root-depth coupling is chosen. + + :param layer: The layer read when ``method`` is ``"layer"``. + """ + + method: Literal["highest_unconfined", "highest_active_head", "layer"] = "highest_unconfined" + layer: LayerNumber | None = None + + +class DynamicRootDepth(_Strict): + """Restriction of the vegetation root depth by the previous step's water table. + + :param minimum_depth_table: Lookup table of the minimum root depth per soil class. + """ + + enabled: bool = False + minimum_depth_table: MapPath | None = None + water_table: WaterTable = WaterTable() + + +class Coupling(_Strict): + dynamic_root_depth: DynamicRootDepth = DynamicRootDepth() + + +class Output(_Strict): + """MODFLOW diagnostics written as rasters.""" + + heads: bool = True + river_leakage: bool = True + storage: bool = False + drain_flow: bool = False + root_depth: bool = False + + +class ModflowSettings(_Strict): + """The MODFLOW section. + + :param enabled: Whether the run is coupled to MODFLOW. + :param top: Raster of the model top, the top of layer 1 [m]. + :param layers: The layers from the top down. + """ + + enabled: bool = False + top: MapPath | None = None + layers: list[ModflowLayer] = [] + dis: Dis = Dis() + solver: Solver = Solver() + wetting: Wetting = Wetting() + river: River = River() + ghb: Ghb = Ghb() + drain: Drain = Drain() + coupling: Coupling = Coupling() + output: Output = Output() + + @model_validator(mode="after") + def _check_enabled(self) -> Self: + if not self.enabled: + return self + errors = [ + *self._structure_errors(), + *self._package_errors(), + *self._storage_errors(), + *self._wetting_errors(), + *self._coupling_errors(), + ] + if errors: + raise ValueError("MODFLOW section: " + " ".join(errors)) + return self + + def _structure_errors(self) -> list[str]: + errors = [] + if not self.top: + errors.append("an enabled section needs the model 'top'.") + if not self.layers: + errors.append("an enabled section needs at least one layer.") + if len(self.layers) > MAX_LAYERS: + errors.append( + f"{len(self.layers)} layers are listed; at most {MAX_LAYERS} layers are " + "supported, because the per-layer outputs mfh, mfst and mfdrn of " + "layer 1 and layer 10 would share file names." + ) + names = [item.name for item in self.layers] + repeated = sorted({name for name in names if names.count(name) > 1}) + if repeated: + errors.append(f"layer names must be unique, repeated: {repeated}.") + for number, item in enumerate(self.layers[1:], start=2): + if item.laycon == 1: + errors.append( + f"layer {number} has LAYCON 1, which is valid only for the top layer (layer 1)." + ) + if not self.river.enabled: + errors.append( + "river.enabled must be true: the RIV leakage is the baseflow of the coupled run." + ) + return errors + + def _range_error(self, what: str, number: int) -> str | None: + count = len(self.layers) + if 1 <= number <= count: + return None + return f"{what} refers to layer {number}, outside the layers 1-{count}." + + def _package_errors(self) -> list[str]: + errors = [] + for name in ("river", "ghb", "drain"): + package = getattr(self, name) + if not package.enabled: + continue + if not package.entries: + errors.append(f"{name}.enabled is true but {name}.entries is empty.") + seen = set() + for index, entry in enumerate(package.entries, start=1): + for number in entry.layers: + error = self._range_error(f"{name} entry {index}", number) + if error: + errors.append(error) + if number in seen: + errors.append(f"{name} lists layer {number} more than once.") + seen.add(number) + if self.river.enabled: + for index, entry in enumerate(self.river.entries, start=1): + if not isinstance(entry.conductance, str) and entry.mask is None: + errors.append( + f"river entry {index} has a numeric conductance and needs a 'mask' " + "raster selecting the river cells." + ) + return errors + + def _storage_errors(self) -> list[str]: + if self.dis.steady_state: + # The extension ends the process when asked for storage in a steady run. + if self.output.storage: + return [ + "output.storage needs a transient run (dis.steady_state false): " + "a steady-state period has no storage flow." + ] + return [] + required = { + 0: ("specific_storage",), + 1: ("specific_yield",), + 2: ("specific_storage", "specific_yield"), + 3: ("specific_storage", "specific_yield"), + } + return [ + f"layer {number} ({item.name}) has LAYCON {item.laycon} and a transient run " + f"needs its '{key}'." + for number, item in enumerate(self.layers, start=1) + for key in required[item.laycon] + if getattr(item, key) is None + ] + + def _wetting_errors(self) -> list[str]: + wetting = self.wetting + if not wetting.enabled: + return [] + errors = [] + if wetting.map is None: + errors.append("wetting is enabled but wetting.map (the WETDRY raster) is not given.") + if wetting.layers is None: + if not self.wetting_layers(): + errors.append("wetting is enabled but no layer has LAYCON 1 or 3.") + return errors + seen = set() + for number in wetting.layers: + if number in seen: + errors.append(f"wetting lists layer {number} more than once.") + seen.add(number) + error = self._range_error("wetting", number) + if error: + errors.append(error) + continue + laycon = self.layers[number - 1].laycon + if laycon not in WETTING_LAYCONS: + errors.append( + f"wetting layer {number} has LAYCON {laycon}; BCF reads WETDRY only for " + "LAYCON 1 or 3." + ) + return errors + + def _coupling_errors(self) -> list[str]: + root_depth = self.coupling.dynamic_root_depth + if not root_depth.enabled: + return [] + errors = [] + if root_depth.minimum_depth_table is None: + errors.append("dynamic_root_depth is enabled but minimum_depth_table is not given.") + water_table = root_depth.water_table + if water_table.method == "layer": + if water_table.layer is None: + errors.append("water_table.method 'layer' needs water_table.layer.") + else: + error = self._range_error("water_table", water_table.layer) + if error: + errors.append(error) + elif water_table.layer is not None: + errors.append("water_table.layer is only given with water_table.method 'layer'.") + if water_table.method == "highest_unconfined" and not any( + item.laycon in (1, 2, 3) for item in self.layers + ): + errors.append( + "water_table.method 'highest_unconfined' needs a layer with LAYCON 1, 2 or 3; " + "use 'highest_active_head' or 'layer' to read a confined head." + ) + return errors + + # ----- layer numbering --------------------------------------------------- + + def pcraster_layer(self, number: int) -> int: + """The PCRaster MODFLOW number (bottom up) of user layer ``number`` (top down). + + :param number: Layer number as configured, 1 being the top layer. + :type number: int + :returns: The number the extension's ``set*``/``get*`` calls take. + :rtype: int + :raises ValueError: If ``number`` is not a configured layer. + """ + self._check_number(number) + return len(self.layers) - number + 1 + + def user_layer(self, pcraster_number: int) -> int: + """The configured number (top down) of PCRaster layer ``pcraster_number`` (bottom up). + + :param pcraster_number: Layer number of the extension, 1 being the bottom layer. + :type pcraster_number: int + :rtype: int + :raises ValueError: If ``pcraster_number`` is not a layer of the model. + """ + self._check_number(pcraster_number) + return len(self.layers) - pcraster_number + 1 + + def _check_number(self, number: int) -> None: + if not 1 <= number <= len(self.layers): + raise ValueError(f"layer {number} is outside the layers 1-{len(self.layers)}.") + + def wetting_layers(self) -> list[int]: + """The user layers (top down) the WETDRY map applies to; empty when wetting is off. + + :rtype: list[int] + """ + if not self.wetting.enabled: + return [] + if self.wetting.layers is not None: + return list(self.wetting.layers) + return [ + number + for number, item in enumerate(self.layers, start=1) + if item.laycon in WETTING_LAYCONS + ] + + # ----- paths ------------------------------------------------------------- + + def resolve_paths(self, base_dir: PathInput | None) -> Self: + """Return a copy whose relative paths are anchored on ``base_dir``. + + Absolute paths and numbers are kept; with ``base_dir=None`` the + settings are returned unchanged. + """ + if base_dir is None: + return self + base = as_path(base_dir) + + def anchor(value: str) -> str: + return value if Path(value).is_absolute() else str(base / value) + + return type(self).model_validate(transform_paths(self.model_dump(mode="json"), anchor)) + + +_LAYER_PATHS = ("bottom", "initial_head", "boundary") +_LAYER_PATHS_OR_NUMBERS = ( + "horizontal_conductivity", + "vertical_conductivity", + "specific_storage", + "specific_yield", +) +_ENTRY_PATHS = { + "river": ("stage", "bottom", "conductance", "mask"), + "ghb": ("head", "conductance"), + "drain": ("elevation", "conductance"), +} + + +def transform_paths(section: dict, transform: Callable[[str], str]) -> dict: + """Apply ``transform`` to every file path of a MODFLOW section document. + + The document is the JSON form of :class:`ModflowSettings` (possibly with + keys left out, as ``exclude_none`` writes it); numbers standing for a + raster are kept, and so are the keys that are not paths. + + :param section: The section as a dictionary; changed in place. + :type section: dict + :param transform: Function applied to each path string. + :type transform: Callable[[str], str] + :returns: ``section``. + :rtype: dict + """ + + def apply(mapping: dict, keys) -> None: + for key in keys: + value = mapping.get(key) + if isinstance(value, str): + mapping[key] = transform(value) + + apply(section, ("top",)) + for layer in section.get("layers", []): + apply(layer, _LAYER_PATHS + _LAYER_PATHS_OR_NUMBERS) + lookup = layer.get("horizontal_conductivity") + if isinstance(lookup, dict): + apply(lookup, ("map", "table")) + apply(section.get("wetting", {}), ("map",)) + for package, keys in _ENTRY_PATHS.items(): + for entry in section.get(package, {}).get("entries", []): + apply(entry, keys) + root_depth = section.get("coupling", {}).get("dynamic_root_depth", {}) + apply(root_depth, ("minimum_depth_table",)) + return section diff --git a/rubem/hydrological_processes/_modflow.py b/rubem/hydrological_processes/_modflow.py new file mode 100644 index 0000000..cbf2dfd --- /dev/null +++ b/rubem/hydrological_processes/_modflow.py @@ -0,0 +1,540 @@ +"""Groundwater flow of the coupled run, through the PCRaster MODFLOW extension. + +RUBEM recharge feeds the MODFLOW RCH package every step (one RUBEM step is one +stress period) and the aquifer-to-river RIV leakage comes back as the baseflow +of the step. General head boundaries (GHB), drains (DRN) and BCF rewetting +shape the heads only. + +The configuration numbers the layers from the top down, like MODFLOW; the +extension numbers them from the bottom up, and +:meth:`~rubem.configuration.modflow_configuration.ModflowSettings.pcraster_layer` +converts. The layers are also given to the extension in its own order, bottom +layer first: it writes the LAYCON of each ``setConductivity`` call to the BCF +file in call order, whatever layer number the call names. Every result is +keyed by the configured (top-down) number. + +The inputs are read once, in :meth:`ModflowGroundwater.initialize`, and the +stress packages are set once: the extension keeps them for every later run. +The value rules live in :mod:`rubem.validation.modflow_inputs`; the module +only refuses what the extension cannot be given at all (missing values on the +cells it uses, a geometry that is not stacked), because the extension ends the +whole process instead of raising on those. +""" + +from __future__ import annotations + +import logging +import math +import shutil +from dataclasses import dataclass, field +from tempfile import TemporaryDirectory + +import numpy as np +import pcraster as pcr +from pcraster._pcraster import Field + +from .._deps import ensure_mf2005_on_path, groundwater_deps_message, missing_groundwater_deps +from .._paths import PathInput, as_path +from ..configuration.modflow_configuration import ConductivityLookup, ModflowLayer, ModflowSettings +from ..file._readers import FieldScale, read_field + +DRY_HEAD = -999.9 +"""Head MODFLOW gives a dry cell (BCF HDRY) [m].""" + +_TIME_UNIT_DAYS = 4 +_LENGTH_UNIT_METRES = 2 +_RECHARGE_TO_HIGHEST_ACTIVE_CELL = 3 + +_STRESS_PACKAGES = ( + ("river", "setRiver", ("stage", "bottom")), + ("ghb", "setGeneralHead", ("head",)), + ("drain", "setDrain", ("elevation",)), +) +"""Package, extension setter and the maps it takes before the conductance.""" + + +def _initialise(clone): + """Start the PCRaster MODFLOW extension on ``clone``. + + :raises RuntimeError: If the extension is not part of the installed pcraster. + """ + try: + from pcraster import initialise + except ImportError as error: + raise RuntimeError(groundwater_deps_message(missing_groundwater_deps() or None)) from error + return initialise(clone) + + +@dataclass(frozen=True) +class ModflowStepResult: + """What one stress period gives back to RUBEM; layers are configured numbers. + + :param baseflow_mm: Aquifer-to-river leakage as a depth over the cell [mm/step]. + :param aquifer_to_river_m3_per_day: RIV leakage from the aquifer [m3/day]. + :param river_to_aquifer_m3_per_day: RIV leakage into the aquifer [m3/day]. + :param net_river_leakage_m3_per_day: Signed RIV leakage, positive into the aquifer [m3/day]. + :param water_table_head: Head of the root-depth coupling [m], ``None`` when it is off. + :param heads: Head per layer [m], when ``output.heads``. + :param storage: Storage flow per layer [m3/day], when ``output.storage``. + :param drain_flow: Signed DRN flow per drain layer, negative out of the aquifer + [m3/day], when ``output.drain_flow``. + """ + + baseflow_mm: Field + aquifer_to_river_m3_per_day: Field + river_to_aquifer_m3_per_day: Field + net_river_leakage_m3_per_day: Field + water_table_head: Field | None = None + heads: dict[int, Field] = field(default_factory=dict) + storage: dict[int, Field] = field(default_factory=dict) + drain_flow: dict[int, Field] = field(default_factory=dict) + + +def _period_days(days: float) -> float: + days = float(days) + if not math.isfinite(days) or days <= 0: + raise ValueError(f"A stress period must last a positive number of days, got {days}.") + return days + + +def _first_cell(cells: np.ndarray) -> str: + row, column = np.argwhere(cells)[0] + 1 + return f"row {row}, column {column}" + + +def _field(values: np.ndarray) -> Field: + return pcr.numpy2pcr(pcr.Scalar, values, np.nan) + + +def _storage_sources(layer: ModflowLayer) -> tuple: + """The BCF ``Sf1`` and ``Sf2`` sources of a layer, by its LAYCON.""" + if layer.laycon == 0: + return layer.specific_storage, layer.specific_storage + if layer.laycon == 1: + return layer.specific_yield, layer.specific_yield + return layer.specific_storage, layer.specific_yield + + +class ModflowGroundwater: + """One MODFLOW model kept for the whole run. + + :param settings: The enabled MODFLOW section, paths resolved. + :type settings: ModflowSettings + :param cell_area_m2: Area of a RUBEM cell [m2]. Its square root gives the + MODFLOW row and column widths whatever the units of the clone, and it + converts the leakage [m3/day] into a depth [mm/step]. + :type cell_area_m2: float + :param run_directory: Directory MODFLOW runs in; its files (``pcrmf.*``) + land there. Created by :meth:`initialize` when absent. + :type run_directory: str | os.PathLike + :param logger: Logger of the run; the module logger by default. + :type logger: logging.Logger | None + :raises ValueError: If the section is not enabled or the area is not a + positive number. + """ + + def __init__( + self, + settings: ModflowSettings, + cell_area_m2: float, + run_directory: PathInput, + logger: logging.Logger | None = None, + ) -> None: + if not settings.enabled: + raise ValueError("The MODFLOW section is not enabled.") + area = float(cell_area_m2) + if not math.isfinite(area) or area <= 0: + raise ValueError(f"The cell area must be finite and positive, got {cell_area_m2} m2.") + self.settings = settings + self.cell_area_m2 = area + self.run_directory = as_path(run_directory) + self.logger = logger or logging.getLogger(__name__) + self.mf = None + self._period = 0 + self._shape: tuple[int, int] = (0, 0) + self._boundaries: dict[int, Field] = {} + self._active: dict[int, np.ndarray] = {} + self._surfaces: list[Field] = [] + self._cells: dict[tuple[str, int], int] = {} + + # ----- initialization ----------------------------------------------------- + + def initialize(self, first_period_days: float) -> None: + """Start the extension and give it every package, once. + + The clone must be set. The packages go in the order the extension + expects: DIS, BAS, BCF, wetting, solver, then RIV, GHB and DRN. + + :param first_period_days: Length of the first stress period [days]. + :type first_period_days: float + :raises RuntimeError: If the model is already initialized, or the + extension or ``mf2005`` is not available. + :raises ValueError: If an input misses values on cells the extension + uses, or the layers are not stacked. + """ + if self.mf is not None: + raise RuntimeError("The MODFLOW model is already initialized.") + days = _period_days(first_period_days) + if ensure_mf2005_on_path() is None: + raise RuntimeError(groundwater_deps_message(missing_groundwater_deps() or None)) + self.run_directory.mkdir(parents=True, exist_ok=True) + clone = pcr.clone() + self._shape = (clone.nrRows(), clone.nrCols()) + self._read_boundaries() + surfaces = self._read_surfaces() + + self.mf = _initialise(clone) + self.logger.info( + "Starting MODFLOW with %d layer(s) in %s.", + len(self.settings.layers), + self.run_directory, + ) + self._set_dis(surfaces, days) + self._set_bas() + self._set_bcf() + self._set_wetting() + self._set_solver() + self._set_stress_packages() + + def _label(self, number: int) -> str: + return f"layer {number} ({self.settings.layers[number - 1].name})" + + def _bottom_up(self): + """``(extension number, configured number, layer)``, bottom layer first. + + The extension takes the per-layer calls in this order: it appends the + LAYCON of each ``setConductivity`` call to the BCF file in call order, + so a top-down loop gives the top layer the LAYCON of the base. + """ + for target in range(1, len(self.settings.layers) + 1): + number = self.settings.user_layer(target) + yield target, number, self.settings.layers[number - 1] + + def _read_boundaries(self) -> None: + for number, layer in enumerate(self.settings.layers, start=1): + boundary = pcr.cover( + pcr.nominal(read_field(layer.boundary, FieldScale.NOMINAL)), pcr.nominal(0) + ) + self._boundaries[number] = boundary + self._active[number] = pcr.pcr2numpy(boundary, 0) != 0 + + def _read_surfaces(self) -> list[Field]: + """The layer surfaces from the bottom up: each bottom from the base, then the top. + + DIS needs elevations over the whole clone. A column inactive in every + layer takes synthetic 1 m layers (elevations 0, 1, 2, ...); the + elevations of the groundwater domain are never filled. + """ + layers = self.settings.layers + domain = np.logical_or.reduce(list(self._active.values())) + sources = [ + (f"The bottom of {self._label(number)}", layers[number - 1].bottom) + for number in range(len(layers), 0, -1) + ] + sources.append(("The model top", self.settings.top)) + surfaces = [] + below = None + for index, (what, path) in enumerate(sources): + values = self._values(path) + self._require(values, domain, what, path, "active columns") + values = np.where(domain, values, float(index)) + if below is not None and (stacked := domain & (values <= below)).any(): + raise ValueError( + f"{what} ({path}) is not above the surface under it in " + f"{int(stacked.sum())} active columns, the first at {_first_cell(stacked)}." + ) + surfaces.append(_field(values)) + below = values + self._surfaces = surfaces + return surfaces + + def _set_dis(self, surfaces: list[Field], days: float) -> None: + self.mf.createBottomLayer(surfaces[0], surfaces[1]) + for surface in surfaces[2:]: + self.mf.addLayer(surface) + # RUBEM's metric square cells, even when the clone is in degrees: the + # rasters are not reprojected. + width = math.sqrt(self.cell_area_m2) + rows, cols = self._shape + self.mf.setRowWidth([width] * rows) + self.mf.setColumnWidth([width] * cols) + dis = self.settings.dis + self.mf.setDISParameter( + _TIME_UNIT_DAYS, + _LENGTH_UNIT_METRES, + days, + dis.nstp, + dis.tsmult, + int(dis.steady_state), + ) + + def _set_bas(self) -> None: + for target, number, layer in self._bottom_up(): + self.mf.setBoundary(self._boundaries[number], target) + head = self._complete( + layer.initial_head, + self._active[number], + 0.0, + f"The initial head of {self._label(number)}", + ) + self.mf.setInitialHead(head, target) + + def _set_bcf(self) -> None: + self.mf.setDryHead(DRY_HEAD) + transient = not self.settings.dis.steady_state + for target, number, layer in self._bottom_up(): + active = self._active[number] + label = self._label(number) + self.mf.setConductivity( + layer.laytype, + self._complete( + layer.horizontal_conductivity, + active, + 1.0, + f"The horizontal conductivity of {label}", + ), + self._complete( + layer.vertical_conductivity, + active, + 1.0, + f"The vertical conductivity of {label}", + ), + target, + layer.compute_conductivity, + ) + if transient: + primary, secondary = _storage_sources(layer) + self.mf.setStorage( + self._complete(primary, active, 0.0, f"The primary storage of {label}"), + self._complete(secondary, active, 0.0, f"The secondary storage of {label}"), + target, + ) + + def _set_wetting(self) -> None: + layers = self.settings.wetting_layers() + if not layers: + return + wetting = self.settings.wetting + self.mf.setWettingParameter(wetting.wetfct, wetting.iwetit, wetting.ihdwet) + # A missing WETDRY value is 0: that cell is never rewetted. + wetdry = self._complete(wetting.map, None, 0.0, "The WETDRY map") + for target in sorted(self.settings.pcraster_layer(number) for number in layers): + self.mf.setWetting(wetdry, target) + + def _set_solver(self) -> None: + solver = self.settings.solver + self.mf.setPCG( + solver.mxiter, + solver.iter1, + solver.npcond, + solver.hclose, + solver.rclose, + solver.relax, + solver.nbpol, + solver.damp, + ) + + def _set_stress_packages(self) -> None: + """Set RIV, GHB and DRN once, with zero conductance outside their cells. + + A cell of a package layer has a positive conductance, lies inside the + mask (a river) and is inside the boundary of the layer. A layer + without cells is not given to the extension, and its getter is never + called (the extension ends the process on that call). + """ + for package, setter, keys in _STRESS_PACKAGES: + settings = getattr(self.settings, package) + if not settings.enabled: + continue + for entry in settings.entries: + conductance = self._values(entry.conductance) + cells = np.isfinite(conductance) & (conductance > 0) + mask = getattr(entry, "mask", None) + if mask is not None: + mask_values = self._values(mask) + cells &= np.isfinite(mask_values) & (mask_values > 0) + maps = {key: self._values(getattr(entry, key)) for key in keys} + for number in entry.layers: + self._set_package_layer( + package, setter, entry, number, conductance, cells, maps + ) + + def _set_package_layer(self, package, setter, entry, number, conductance, cells, maps) -> None: + layer_cells = cells & self._active[number] + count = int(layer_cells.sum()) + self._cells[(package, number)] = count + if not count: + self.logger.warning( + "The MODFLOW %s package has no cell in %s; that layer is left out.", + package, + self._label(number), + ) + return + stress = [ + self._fill( + values, + layer_cells, + 0.0, + f"The {package} {key} of {self._label(number)}", + getattr(entry, key), + ) + for key, values in maps.items() + ] + getattr(self.mf, setter)( + *stress, + _field(np.where(layer_cells, conductance, 0.0)), + self.settings.pcraster_layer(number), + ) + + # ----- reading ------------------------------------------------------------ + + def _values(self, source) -> np.ndarray: + """A raster, a class lookup or a number as values over the clone (missing: NaN).""" + if isinstance(source, ConductivityLookup): + return pcr.pcr2numpy(self._lookup(source), np.nan) + if isinstance(source, str): + return pcr.pcr2numpy(pcr.scalar(read_field(source, FieldScale.SCALAR)), np.nan) + return np.full(self._shape, float(source)) + + def _lookup(self, source: ConductivityLookup) -> Field: + """The table value of each class, read from a fresh copy of the table. + + PCRaster caches lookup tables by file name, and a calibration rewrites + the table between the runs of one process. + """ + classes = read_field(source.map, FieldScale.NOMINAL) + with TemporaryDirectory(prefix="kh_", dir=self.run_directory) as directory: + snapshot = as_path(directory) / "conductivity.tbl" + shutil.copyfile(source.table, snapshot) + return pcr.lookupscalar(str(snapshot), pcr.nominal(classes)) + + def _complete(self, source, required, fill: float, what: str) -> Field: + return self._fill(self._values(source), required, fill, what, source) + + def _fill(self, values, required, fill: float, what: str, source) -> Field: + """``values`` with its missing cells filled; ``required`` cells may not be missing.""" + if required is not None: + self._require(values, required, what, source, "cells it is used on") + return _field(np.where(np.isfinite(values), values, fill)) + + @staticmethod + def _require(values, cells, what: str, source, where: str) -> None: + missing = cells & ~np.isfinite(values) + if missing.any(): + file = source.map if isinstance(source, ConductivityLookup) else source + raise ValueError( + f"{what} ({file}) has missing values on {int(missing.sum())} {where}, " + f"the first at {_first_cell(missing)}." + ) + + # ----- stress periods ----------------------------------------------------- + + def run_step(self, recharge_mm: Field, days_in_period: float) -> ModflowStepResult: + """Run one stress period with the recharge of the RUBEM step. + + The water-table head is the head of this period; the root-depth + coupling applies it to the next step. + + :param recharge_mm: Recharge of the step [mm/step]; missing cells get none. + :type recharge_mm: Field + :param days_in_period: Length of the step [days]. + :type days_in_period: float + :rtype: ModflowStepResult + :raises RuntimeError: If the model is not initialized, or MODFLOW does + not converge (the next run of the extension would end the process). + """ + if self.mf is None: + raise RuntimeError("initialize() must run before the first MODFLOW step.") + days = _period_days(days_in_period) + self._period += 1 + dis = self.settings.dis + if not dis.steady_state: + self.mf.updateDISParameter(days, dis.nstp, dis.tsmult) + recharge = pcr.cover(pcr.scalar(recharge_mm) / (1000.0 * days), pcr.scalar(0.0)) + self.mf.setRecharge(recharge, _RECHARGE_TO_HIGHEST_ACTIVE_CELL) + self.mf.run(str(self.run_directory)) + if not self.mf.converged(): + raise RuntimeError( + f"MODFLOW did not converge in stress period {self._period}; " + f"see {self.run_directory / 'pcrmf.lst'}." + ) + + net, aquifer_to_river, river_to_aquifer = self._river_exchange() + output = self.settings.output + root_depth = self.settings.coupling.dynamic_root_depth.enabled + heads = self._by_layer(self.mf.getHeads) if output.heads or root_depth else {} + return ModflowStepResult( + baseflow_mm=aquifer_to_river * days * 1000.0 / self.cell_area_m2, + aquifer_to_river_m3_per_day=aquifer_to_river, + river_to_aquifer_m3_per_day=river_to_aquifer, + net_river_leakage_m3_per_day=net, + water_table_head=self._water_table_head(heads) if root_depth else None, + heads=heads if output.heads else {}, + storage=self._by_layer(self.mf.getStorage) if output.storage else {}, + drain_flow=self._drain_flow() if output.drain_flow else {}, + ) + + def _by_layer(self, getter) -> dict[int, Field]: + return { + number: pcr.scalar(getter(self.settings.pcraster_layer(number))) + for number in range(1, len(self.settings.layers) + 1) + } + + def _layers_with_cells(self, package: str) -> list[int]: + return [ + number for (name, number), count in self._cells.items() if name == package and count + ] + + def _river_exchange(self) -> tuple[Field, Field, Field]: + """Net, aquifer-to-river and river-to-aquifer leakage summed over the river layers.""" + zero = pcr.spatial(pcr.scalar(0.0)) + net, aquifer_to_river, river_to_aquifer = zero, zero, zero + for number in self._layers_with_cells("river"): + leakage = pcr.scalar(self.mf.getRiverLeakage(self.settings.pcraster_layer(number))) + net = net + leakage + # Negative leakage leaves the aquifer: that is the baseflow. + aquifer_to_river = aquifer_to_river + pcr.max(-leakage, 0.0) + river_to_aquifer = river_to_aquifer + pcr.max(leakage, 0.0) + return net, aquifer_to_river, river_to_aquifer + + def _drain_flow(self) -> dict[int, Field]: + if not self.settings.drain.enabled: + return {} + with_cells = set(self._layers_with_cells("drain")) + return { + number: ( + pcr.scalar(self.mf.getDrain(self.settings.pcraster_layer(number))) + if number in with_cells + else pcr.spatial(pcr.scalar(0.0)) + ) + for entry in self.settings.drain.entries + for number in entry.layers + } + + def _water_table_head(self, heads: dict[int, Field]) -> Field: + """The head of the root-depth coupling, by the configured method. + + ``layer`` takes that layer; the other methods take, cell by cell, the + first valid head from the top down (dry and inactive cells are not + valid); ``highest_unconfined`` also skips confined layers (LAYCON 0) + and convertible ones (LAYCON 2 and 3) where the head is above the top + of the layer. + """ + water_table = self.settings.coupling.dynamic_root_depth.water_table + if water_table.method == "layer": + return self._valid_head(water_table.layer, heads) + selected = None + for number, layer in enumerate(self.settings.layers, start=1): + if water_table.method == "highest_unconfined" and layer.laycon == 0: + continue + candidate = self._valid_head(number, heads) + if water_table.method == "highest_unconfined" and layer.laycon in (2, 3): + top = self._surfaces[self.settings.pcraster_layer(number)] + candidate = pcr.ifthen(candidate <= top, candidate) + selected = candidate if selected is None else pcr.cover(selected, candidate) + return selected + + def _valid_head(self, number: int, heads: dict[int, Field]) -> Field: + head = heads[number] + valid = pcr.defined(head) & (self._boundaries[number] != 0) & (head != DRY_HEAD) + return pcr.ifthen(valid, head) diff --git a/rubem/validation/lookup_tables.py b/rubem/validation/lookup_tables.py index 42a9102..dc44249 100644 --- a/rubem/validation/lookup_tables.py +++ b/rubem/validation/lookup_tables.py @@ -95,6 +95,36 @@ def _in_interval(value: float, interval: str) -> bool: return low_ok and high_ok +def _first_matching_row(rows, value: float) -> int | None: + """The index of the first row of a lookup table whose key matches ``value``. + + PCRaster reads that row for a cell of the class ``value``: a numeric key + matches an equal number, an interval key every number it contains, and a + later row of the same class is never read. + + :param rows: The rows of a one-key table, as :func:`read_lookup_table` returns them. + :type rows: list[tuple[tuple[str], float]] + + :param value: The class of a cell. + :type value: float + + :return: The index of the row, ``None`` when no row matches. + :rtype: int | None + """ + for index, ((key,), _) in enumerate(rows): + if _NUMBER.match(key) and float(key) == value: + return index + if _INTERVAL.match(key) and _in_interval(value, key): + return index + return None + + +def _table_value(rows, value: float) -> float | None: + """The value PCRaster reads from a lookup table for the class ``value``, ``None`` if none.""" + index = _first_matching_row(rows, value) + return None if index is None else rows[index][1] + + def _canonical(key: str) -> str: """Canonical spelling of a key, so that ``1``, ``01`` and ``1.0`` are one class. diff --git a/rubem/validation/modflow_inputs.py b/rubem/validation/modflow_inputs.py new file mode 100644 index 0000000..5fd1bc4 --- /dev/null +++ b/rubem/validation/modflow_inputs.py @@ -0,0 +1,683 @@ +"""Checks of the MODFLOW inputs and runtime, run before a coupled simulation. + +:func:`check_modflow_inputs` is called by +:class:`~rubem.configuration.model_configuration.ModelConfiguration` when the +MODFLOW section is enabled. The files the run reads must exist and the +MODFLOW runtime must be installed whatever ``validate_input`` says: without +them the coupled run cannot start, and a calibration would only find out in +its workers. The content of the rasters and tables is checked with +``validate_input``, as the other inputs are. + +The content checks read every raster the way the run does, through +:func:`~rubem.file._readers.read_field` on the clone of the configuration, so +they make that clone the PCRaster clone of the process (the one the model sets +again when it starts). PCRaster reads a map of another size on the clone +without complaint, so the geometry of every raster is compared with the +clone's first. Layers are named by their configured number, 1 being the top +layer. The cross-field rules of the section (layer numbers, LAYCON of the +wetting layers, storage keys) are checked by +:class:`~rubem.configuration.modflow_configuration.ModflowSettings` itself. +""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import TYPE_CHECKING + +import numpy as np + +from .._deps import groundwater_deps_message, missing_groundwater_deps, missing_runtime_deps +from ..configuration._problems import Problem +from ..configuration.modflow_configuration import ConductivityLookup, transform_paths +from ..file._readers import FieldScale, is_geotiff, read_field, set_clone +from .lookup_tables import LookupTableError, _table_value, read_lookup_table + +if TYPE_CHECKING: + from ..configuration.input_raster_files import InputRasterFiles + from ..configuration.input_table_files import InputTableFiles + from ..configuration.modflow_configuration import ModflowSettings + +_BOUNDARY_VALUES = (-1, 0, 1) +_RASTER_IMPLICATION = "The simulation cannot run with this raster." +_TABLE_IMPLICATION = "The simulation cannot run with this table." +_PACKAGE_STRESS_KEYS = {"river": ("stage", "bottom"), "ghb": ("head",), "drain": ("elevation",)} + + +def check_modflow_inputs( + settings: ModflowSettings, + raster_files: InputRasterFiles, + validate_input: bool, + *, + tables: InputTableFiles | None = None, +) -> list[Problem]: + """Check the inputs of an enabled MODFLOW section and the MODFLOW runtime. + + Always, blocking: every file the run reads exists and is not empty (the + files of a disabled package, of disabled wetting and of a disabled + root-depth coupling are not read); the PCRaster MODFLOW extension and the + ``mf2005`` executable are available (see + :func:`~rubem._deps.missing_groundwater_deps`). + + With ``validate_input``, when every file exists and PCRaster is installed: + + - blocking: a raster that cannot be read or does not share the clone + geometry; boundary values other than -1, 0 and 1; a model top or layer + bottom missing on a column active in any layer, or a bottom not strictly + below the top of its layer there; an initial head, conductivity or (in + a transient run) storage map missing on an active cell of its layer; an + initial head below the layer bottom in every active cell; a conductivity + class map that is not nominal, or a class on an active cell without a + positive value in the table; an enabled package layer without a cell of + positive conductance inside the boundary of the layer (the PCRaster + MODFLOW extension aborts the process on it); a package head or + elevation missing on a cell of the package; with the root-depth + coupling, a minimum root depth table that does not cover the soil + classes or whose value is not in ``(0, Zr]``; + - warning: an initial head below the layer bottom in some active cells; + river cells whose bed lies outside the elevation interval of their layer. + + :param settings: The MODFLOW section, paths already anchored. + :type settings: ModflowSettings + :param raster_files: The input rasters of the configuration (clone, soil). + :type raster_files: InputRasterFiles + :param validate_input: Whether to check the content of the inputs. + :type validate_input: bool + :param tables: The lookup tables of the configuration; their rootzone + depth (Zr) bounds the minimum root depth. Defaults to ``None``, which + skips that comparison. + :type tables: InputTableFiles | None + :returns: The problems found, blocking ones flagged. + :rtype: list[Problem] + """ + problems = _check_files(settings) + files_unusable = bool(problems) + missing = missing_groundwater_deps() + if missing: + problems.append( + Problem( + description="MODFLOW runtime is not available.", + reason=groundwater_deps_message(missing), + implication="The coupled simulation cannot start.", + blocking=True, + ) + ) + if not validate_input or files_unusable or "pcraster" in missing_runtime_deps(): + return problems + return problems + _ContentChecks(settings, raster_files, tables).run() + + +def _read_files(settings: ModflowSettings) -> list[str]: + """The files the coupled run reads, in the order of the section, without repeats.""" + document = settings.model_dump(mode="json") + if not settings.wetting.enabled: + document.pop("wetting") + for package in ("river", "ghb", "drain"): + if not getattr(settings, package).enabled: + document.pop(package) + if not settings.coupling.dynamic_root_depth.enabled: + document.pop("coupling") + files: list[str] = [] + + def collect(path: str) -> str: + if path not in files: + files.append(path) + return path + + transform_paths(document, collect) + return files + + +def _check_files(settings: ModflowSettings) -> list[Problem]: + problems = [] + for path in _read_files(settings): + file = Path(path) + if not file.is_file(): + problems.append( + Problem( + description="MODFLOW input file does not exist.", + reason="The MODFLOW section names this file and the coupled run reads it.", + implication="The simulation cannot run without it.", + file=path, + blocking=True, + ) + ) + elif file.stat().st_size <= 0: + problems.append( + Problem( + description="MODFLOW input file is empty.", + reason="The MODFLOW section names this file and the coupled run reads it.", + implication="The simulation cannot run with it.", + file=path, + blocking=True, + ) + ) + return problems + + +_HEAD_SENTINELS = (-888.0, -999.0, -999.9, -999.99, -9999.0) +"""Values MODFLOW tools write for no-flow or dry cells (HNOFLO, HDRY) and common no-data markers.""" + + +def _first_cell(cells: np.ndarray) -> str: + row, column = (int(index) + 1 for index in np.argwhere(cells)[0]) + return f"row {row}, column {column}" + + +def _numbers(values) -> list: + """Values for a message: integers without a decimal part, the rest as floats.""" + return [int(value) if float(value).is_integer() else float(value) for value in values] + + +def _blocking(description: str, reason: str, file, implication=_RASTER_IMPLICATION) -> Problem: + return Problem( + description=description, + reason=reason, + implication=implication, + file=str(file) if file is not None else None, + blocking=True, + ) + + +class _ContentChecks: + """The content rules of :func:`check_modflow_inputs`; rasters are read once.""" + + def __init__(self, settings, raster_files, tables) -> None: + self.settings = settings + self.raster_files = raster_files + self.tables = tables + self.problems: list[Problem] = [] + self.fields: dict = {} + self.arrays: dict = {} + self.clone_geometry = None + self.active: dict[int, np.ndarray] = {} + self.bottoms: dict[int, np.ndarray | None] = {} + self.tops: dict[int, np.ndarray | None] = {} + + def run(self) -> list[Problem]: + from ..configuration.output_raster_base import read_raster_geometry, reference_crs + + clone = self.raster_files.clone + set_clone( + clone, + projection=reference_crs(clone, self.raster_files.georeference, self.raster_files.dem), + ) + cols, rows, transform, _ = read_raster_geometry(clone) + self.clone_geometry = (cols, rows, transform) + self.shape = (rows, cols) + if not self._check_boundaries(): + return self.problems + self._check_geometry() + for number in range(1, len(self.settings.layers) + 1): + self._check_layer(number) + for package in ("river", "ghb", "drain"): + if getattr(self.settings, package).enabled: + self._check_package(package) + if self.settings.coupling.dynamic_root_depth.enabled: + self._check_minimum_root_depth() + return self.problems + + # ----- reading ------------------------------------------------------------ + + def _label(self, number: int) -> str: + return f"layer {number} ({self.settings.layers[number - 1].name})" + + def _field(self, path: str, scale: FieldScale = FieldScale.SCALAR): + """The raster as a field on the clone, or ``None`` once its problem is reported.""" + key = (path, scale) + if key in self.fields: + return self.fields[key] + from ..configuration.output_raster_base import read_raster_geometry + + field = None + try: + cols, rows, transform, _ = read_raster_geometry(path) + clone_cols, clone_rows, clone_transform = self.clone_geometry + if (cols, rows) != (clone_cols, clone_rows) or not all( + abs(a - b) <= 1e-9 * max(1.0, abs(a)) + for a, b in zip(transform, clone_transform, strict=True) + ): + self.problems.append( + _blocking( + "MODFLOW raster does not share the clone geometry.", + f"{cols}x{rows} cells with transform {transform}; the clone has " + f"{clone_cols}x{clone_rows} with {clone_transform}.", + path, + ) + ) + else: + field = read_field(path, scale) + except (OSError, RuntimeError, ValueError) as error: + self.problems.append(_blocking("MODFLOW raster cannot be read.", str(error), path)) + self.fields[key] = field + return field + + def _values(self, source) -> np.ndarray | None: + """A scalar raster (missing cells as NaN), or a number spread over the clone.""" + if source is None: + return None + if not isinstance(source, str): + return np.full(self.shape, float(source)) + if source not in self.arrays: + import pcraster as pcr + + field = self._field(source) + self.arrays[source] = ( + None if field is None else pcr.pcr2numpy(pcr.scalar(field), np.nan) + ) + return self.arrays[source] + + def _require_finite(self, values, cells, description: str, where: str, file) -> bool: + """Report ``values`` missing on ``cells``; ``where`` names the cells.""" + invalid = cells & ~np.isfinite(values) + if not invalid.any(): + return True + self.problems.append( + _blocking( + description, + f"{int(invalid.sum())} of {int(cells.sum())} {where}, the first at " + f"{_first_cell(invalid)}.", + file, + ) + ) + return False + + # ----- layers ------------------------------------------------------------- + + def _check_boundaries(self) -> bool: + readable = True + for number, item in enumerate(self.settings.layers, start=1): + values = self._values(item.boundary) + if values is None: + readable = False + continue + finite = np.isfinite(values) + invalid = sorted(set(values[finite].tolist()) - set(_BOUNDARY_VALUES)) + if invalid: + self.problems.append( + _blocking( + f"MODFLOW boundary of {self._label(number)} has invalid values.", + "Values other than -1 (constant head), 0 (inactive) and 1 (active): " + f"{_numbers(invalid[:10])}.", + item.boundary, + ) + ) + self.active[number] = finite & (values != 0) + return readable + + def _check_geometry(self) -> None: + domain = np.logical_or.reduce(list(self.active.values())) + layers = self.settings.layers + surfaces = [("MODFLOW model top", self.settings.top)] + [ + (f"MODFLOW bottom of {self._label(number)}", item.bottom) + for number, item in enumerate(layers, start=1) + ] + values = [] + for label, path in surfaces: + surface = self._values(path) + if surface is not None and not self._require_finite( + surface, + domain, + f"{label} has missing values on active columns.", + "columns active in a layer", + path, + ): + surface = None + values.append(surface) + for number in range(1, len(layers) + 1): + self.tops[number], self.bottoms[number] = values[number - 1], values[number] + upper, lower = values[number - 1], values[number] + if upper is None or lower is None: + continue + invalid = domain & ~(lower < upper) + if invalid.any(): + above = "the model top" if number == 1 else f"the bottom of layer {number - 1}" + self.problems.append( + _blocking( + f"MODFLOW bottom of {self._label(number)} is not below the top of " + "the layer.", + f"The top of the layer is {above}; {int(invalid.sum())} cell(s) of the " + f"columns active in a layer are not strictly below it, the first at " + f"{_first_cell(invalid)}.", + layers[number - 1].bottom, + ) + ) + + def _check_layer(self, number: int) -> None: + item = self.settings.layers[number - 1] + active = self.active[number] + label = self._label(number) + where = "active cells of the layer" + head = self._values(item.initial_head) + if head is not None and self._require_finite( + head, + active, + f"MODFLOW initial head of {label} has missing values on active cells.", + where, + item.initial_head, + ): + sentinels = self._check_head_sentinels(number, head) + self._check_head_above_bottom(number, head, ignore=sentinels) + properties = [ + ("vertical conductivity", item.vertical_conductivity), + ("horizontal conductivity", item.horizontal_conductivity), + ] + if not self.settings.dis.steady_state: + properties += [ + ("specific storage", item.specific_storage), + ("specific yield", item.specific_yield), + ] + for name, source in properties: + if isinstance(source, ConductivityLookup): + self._check_conductivity_classes(number, source) + elif isinstance(source, str): + values = self._values(source) + if values is not None: + self._require_finite( + values, + active, + f"MODFLOW {name} of {label} has missing values on active cells.", + where, + source, + ) + + def _check_head_sentinels(self, number: int, head: np.ndarray) -> np.ndarray: + """Report no-data markers read as heads; return the cells that hold them.""" + active = self.active[number] + found: list[float] = [] + sentinel = np.zeros(head.shape, dtype=bool) + for value in _HEAD_SENTINELS: + cells = active & np.isclose(head, value, rtol=0.0, atol=1e-3) + if cells.any(): + found.append(value) + sentinel |= cells + if not found: + return sentinel + item = self.settings.layers[number - 1] + values = " and ".join(f"{value:g}" for value in found) + markers = ( + "a MODFLOW no-data marker, not a head" + if len(found) == 1 + else ("MODFLOW no-data markers, not heads") + ) + self.problems.append( + _blocking( + f"MODFLOW initial head of {self._label(number)} holds a no-data sentinel in " + "active cells.", + f"{int(sentinel.sum())} of {int(active.sum())} active cells hold the " + f"value{'s' if len(found) > 1 else ''} {values}, {markers}; the first at " + f"{_first_cell(sentinel)}.", + item.initial_head, + implication=( + "MODFLOW reads the marker as a head: those cells start dry and the value " + "reaches the head outputs. Give them a head or make them inactive." + ), + ) + ) + return sentinel + + def _check_head_above_bottom( + self, number: int, head: np.ndarray, ignore: np.ndarray | None = None + ) -> None: + bottom = self.bottoms.get(number) + if bottom is None: + return + cells = self.active[number] & np.isfinite(bottom) + if ignore is not None: + cells &= ~ignore + below = cells & (head < bottom) + count, total = int(below.sum()), int(cells.sum()) + if not count: + return + item = self.settings.layers[number - 1] + reason = ( + f"{count} of {total} active cells have an initial head below the bottom of the " + f"layer, the first at {_first_cell(below)}." + ) + if count == total: + self.problems.append( + _blocking( + f"MODFLOW initial head of {self._label(number)} is below the layer bottom " + "in every active cell.", + reason, + item.initial_head, + implication="MODFLOW would start with the whole layer dry.", + ) + ) + return + self.problems.append( + Problem( + description=f"MODFLOW initial head of {self._label(number)} is below the layer " + "bottom in some active cells.", + reason=reason, + implication="MODFLOW starts with those cells dry.", + file=item.initial_head, + ) + ) + + def _check_conductivity_classes(self, number: int, lookup: ConductivityLookup) -> None: + import pcraster as pcr + + label = self._label(number) + field = self._field(lookup.map, FieldScale.NOMINAL) + if field is None: + return + if not is_geotiff(lookup.map) and field.dataType() != pcr.Nominal: + self.problems.append( + _blocking( + f"MODFLOW horizontal conductivity classes of {label} must be a nominal map.", + f"The map has the {field.dataType()} value scale; its classes are looked " + "up in the table.", + lookup.map, + ) + ) + return + classes = pcr.pcr2numpy(pcr.scalar(field), np.nan) + active = self.active[number] + if not self._require_finite( + classes, + active, + f"MODFLOW horizontal conductivity classes of {label} have missing values on " + "active cells.", + "active cells of the layer", + lookup.map, + ): + return + try: + rows = read_lookup_table(lookup.table) + except (OSError, LookupTableError) as error: + self.problems.append( + _blocking( + f"MODFLOW horizontal conductivity table of {label} cannot be read.", + str(error), + lookup.table, + implication=_TABLE_IMPLICATION, + ) + ) + return + uncovered, invalid = [], [] + for value in sorted(set(classes[active].tolist())): + conductivity = _table_value(rows, value) + if conductivity is None: + uncovered.append(value) + elif not (math.isfinite(conductivity) and conductivity > 0): + invalid.append(value) + if uncovered: + self.problems.append( + _blocking( + f"MODFLOW horizontal conductivity table of {label} does not cover the " + "classes of its map.", + f"Classes on active cells without a row: {_numbers(uncovered)}.", + lookup.table, + implication=_TABLE_IMPLICATION, + ) + ) + if invalid: + self.problems.append( + _blocking( + f"MODFLOW horizontal conductivity table of {label} has non-positive values.", + f"Classes on active cells whose conductivity is not positive: " + f"{_numbers(invalid)}.", + lookup.table, + implication=_TABLE_IMPLICATION, + ) + ) + + # ----- packages ----------------------------------------------------------- + + def _package_cells(self, entry) -> np.ndarray | None: + """The cells of positive conductance (inside the mask, for a river), any layer.""" + conductance = self._values(entry.conductance) + mask = self._values(getattr(entry, "mask", None)) + if conductance is None or (getattr(entry, "mask", None) and mask is None): + return None + cells = np.isfinite(conductance) & (conductance > 0) + if mask is not None: + cells &= np.isfinite(mask) & (mask > 0) + return cells + + def _check_package(self, package: str) -> None: + for entry in getattr(self.settings, package).entries: + cells = self._package_cells(entry) + if cells is None: + continue + used = np.zeros(self.shape, dtype=bool) + for number in entry.layers: + layer_cells = cells & self.active[number] + if not layer_cells.any(): + self.problems.append( + _blocking( + f"MODFLOW {package} package has no cell in {self._label(number)}.", + "No active cell of the layer has a positive conductance" + + (" inside the mask" if getattr(entry, "mask", None) else "") + + "; cells outside the boundary of the layer do not count.", + entry.conductance if isinstance(entry.conductance, str) else entry.mask, + implication="The PCRaster MODFLOW extension aborts the process " + "when it reads a package layer without cells.", + ) + ) + used |= layer_cells + complete = True + for key in _PACKAGE_STRESS_KEYS[package]: + path = getattr(entry, key) + values = self._values(path) + complete &= values is not None and self._require_finite( + values, + used, + f"MODFLOW {package} {key} has missing values on the cells of the package.", + "cells of the package", + path, + ) + if package == "river" and complete: + self._check_riverbed(entry, cells) + + def _check_riverbed(self, entry, cells: np.ndarray) -> None: + bed = self._values(entry.bottom) + for number in entry.layers: + bottom, top = self.bottoms.get(number), self.tops.get(number) + if bottom is None or top is None: + continue + river = cells & self.active[number] + below = river & (bed < bottom) + above = river & (bed > top) + outside = int(below.sum() + above.sum()) + if not outside: + continue + self.problems.append( + Problem( + description="MODFLOW river bed lies outside the elevation interval of its " + "layer.", + reason=f"{outside} of {int(river.sum())} river cells of layer {number} have " + "their bed outside the layer's elevation interval: " + f"{int(below.sum())} below the layer bottom, {int(above.sum())} above the " + "layer top.", + implication="The river exchange of those cells is computed in a layer " + "that does not hold the riverbed; moving them to another layer (a second " + "river entry with its own mask) is a modelling decision.", + file=entry.bottom, + ) + ) + + # ----- coupling ----------------------------------------------------------- + + def _check_minimum_root_depth(self) -> None: + import pcraster as pcr + + label = "Minimum root depth (Dpz_min)" + path = self.settings.coupling.dynamic_root_depth.minimum_depth_table + try: + rows = read_lookup_table(path) + except (OSError, LookupTableError) as error: + self.problems.append( + _blocking( + f"{label} lookup table cannot be read.", + str(error), + path, + implication=_TABLE_IMPLICATION, + ) + ) + return + soil = self._field(self.raster_files.soil, FieldScale.NOMINAL) + if soil is None: + return + values = pcr.pcr2numpy(pcr.scalar(soil), np.nan) + classes = sorted(set(values[np.isfinite(values)].tolist())) + depths = {value: _table_value(rows, value) for value in classes} + uncovered = [value for value, depth in depths.items() if depth is None] + if uncovered: + self.problems.append( + _blocking( + f"{label} lookup table does not cover the soil classes.", + f"Soil classes without a row: {_numbers(uncovered)}.", + path, + implication=_TABLE_IMPLICATION, + ) + ) + non_positive = [ + (value, depth) + for value, depth in depths.items() + if depth is not None and not (math.isfinite(depth) and depth > 0) + ] + if non_positive: + self.problems.append( + _blocking( + f"{label} lookup table has non-positive values.", + "Values <= 0: " + + "; ".join( + f"soil class {_numbers([value])[0]}: {depth:g}" + for value, depth in non_positive + ) + + ".", + path, + implication=_TABLE_IMPLICATION, + ) + ) + exceeding = self._above_rootzone_depth(depths) + if exceeding: + self.problems.append( + _blocking( + f"{label} exceeds the rootzone depth (Zr).", + "; ".join( + f"soil class {_numbers([value])[0]}: Dpz_min {depth:g} > Zr {zr:g}" + for value, depth, zr in exceeding + ) + + ".", + path, + implication=_TABLE_IMPLICATION, + ) + ) + + def _above_rootzone_depth(self, depths: dict) -> list[tuple[float, float, float]]: + """The soil classes whose minimum root depth exceeds Zr (unreadable Zr: none).""" + if self.tables is None: + return [] + try: + rows = read_lookup_table(self.tables.rootzone_depth) + except (OSError, LookupTableError): + return [] # Reported by check_lookup_tables. + exceeding = [] + for value, depth in depths.items(): + zr = _table_value(rows, value) + if depth is not None and zr is not None and depth > zr: + exceeding.append((value, depth, zr)) + return exceeding diff --git a/tests/helpers/synthetic.py b/tests/helpers/synthetic.py index a15dc40..d53ffb3 100644 --- a/tests/helpers/synthetic.py +++ b/tests/helpers/synthetic.py @@ -48,6 +48,8 @@ class of the regression fixtures) but tiny, so a full model run takes "Tw": 0.12, "Zr": 150.39, } +ROOTZONE_DEPTH = _SOIL_TABLES["Zr"] +"""Rootzone depth (``Zr``) of the soil class [cm].""" def _as_tif(path): @@ -319,3 +321,131 @@ def write_lai_max_table(config, values=None): for lulc_class, value in values.items(): f.write(f"{lulc_class} {value}\n") return path + + +MODFLOW_TOP = 100.0 +MODFLOW_LAYER_THICKNESS = 20.0 +MODFLOW_HEAD = 90.0 +MODFLOW_KH_TABLE = {1: 0.5, 2: 0.1} + + +def write_grid_map(path, values, nominal=False): + """Write ``values`` (nine numbers, row by row) as a PCRaster map on the synthetic grid. + + :param path: The map file; its directory is created. + :param values: A scalar broadcast to the grid, or ``ROWS * COLS`` values; + ``NaN`` and :data:`MISSING` cells are missing. + :param nominal: Whether the map is nominal (classes) instead of scalar. + :returns: ``path`` as a string. + """ + import pcraster as pcr + + pcr.setclone(ROWS, COLS, CELL_SIZE, WEST, NORTH) + array = np.broadcast_to(np.asarray(values, dtype=np.float64), (ROWS * COLS,)) + array = np.where(np.isnan(array), MISSING, array).reshape(ROWS, COLS) + os.makedirs(os.path.dirname(str(path)), exist_ok=True) + if nominal: + field = pcr.numpy2pcr(pcr.Nominal, array.astype(np.int32), int(MISSING)) + else: + field = pcr.numpy2pcr(pcr.Scalar, array.astype(np.float32), MISSING) + pcr.report(field, str(path)) + return str(path) + + +def write_modflow_inputs(config, layers=3): + """Write the inputs of a MODFLOW section on the synthetic grid and return the section. + + The layers are listed top down under a flat model top at + :data:`MODFLOW_TOP`, each :data:`MODFLOW_LAYER_THICKNESS` thick and active + everywhere, with the initial head :data:`MODFLOW_HEAD` in every layer. + Layer 1 is unconfined (LAYCON 1) with its horizontal conductivity given by + classes (row 1 class 1, the other rows class 2, see + :data:`MODFLOW_KH_TABLE`) and its vertical conductivity by a map; the other + layers are convertible (LAYCON 2) with a conductivity map and a number. The + river lies on the middle column of layer 1 (numeric conductance and a + mask), the general head boundary on the left column of every layer, the + drain package is off and wetting applies to layer 1. + + :param config: The configuration returned by :func:`write_synthetic_dataset`; + the files go to ``maps/modflow`` next to its rasters. + :param layers: Number of layers. + :returns: The ``MODFLOW`` section, with absolute paths. + """ + maps = os.path.dirname(os.path.dirname(config["RASTERS"]["clone"])) + directory = os.path.join(maps, "modflow") + + def grid(name, values, nominal=False): + return write_grid_map(os.path.join(directory, name), values, nominal) + + left_column = [1.0 if index % COLS == 0 else 0.0 for index in range(ROWS * COLS)] + middle_column = [1.0 if index % COLS == 1 else 0.0 for index in range(ROWS * COLS)] + table = os.path.join(directory, "kh1.tbl") + os.makedirs(directory, exist_ok=True) + with open(table, "w", encoding="utf8") as f: + for kh_class, value in MODFLOW_KH_TABLE.items(): + f.write(f"{kh_class} {value}\n") + boundary = grid("bound.map", 1, nominal=True) + head = grid("head.map", MODFLOW_HEAD) + section_layers = [] + for number in range(1, layers + 1): + item = { + "name": f"layer{number}", + "bottom": grid(f"bottom{number}.map", MODFLOW_TOP - number * MODFLOW_LAYER_THICKNESS), + "initial_head": head, + "boundary": boundary, + "laytype": 2, + "horizontal_conductivity": grid("kh.map", 1.0), + "vertical_conductivity": 0.1, + "specific_yield": 0.15, + "specific_storage": 1e-5, + } + if number == 1: + classes = [1] * COLS + [2] * (ROWS - 1) * COLS + item["laytype"] = 1 + item["horizontal_conductivity"] = { + "map": grid("kh_classes1.map", classes, nominal=True), + "table": table, + } + item["vertical_conductivity"] = grid("kv1.map", 0.1) + section_layers.append(item) + return { + "enabled": True, + "top": grid("top.map", MODFLOW_TOP), + "layers": section_layers, + "wetting": {"enabled": True, "map": grid("wet.map", 1.0), "layers": [1]}, + "river": { + "enabled": True, + "entries": [ + { + "layers": [1], + "stage": grid("riv_stage.map", MODFLOW_HEAD + 5.0), + "bottom": grid("riv_bottom.map", MODFLOW_TOP - 10.0), + "conductance": 10.0, + "mask": grid("riv_mask.map", middle_column), + } + ], + }, + "ghb": { + "enabled": True, + "entries": [ + { + "layers": list(range(1, layers + 1)), + "head": grid("ghb_head.map", MODFLOW_HEAD), + "conductance": grid("ghb_cond.map", left_column), + } + ], + }, + "drain": {"enabled": False, "entries": []}, + } + + +def write_minimum_root_depth_table(config, value): + """Write a ``Dpz_min`` lookup table next to the soil tables and return its path. + + :param config: The configuration returned by :func:`write_synthetic_dataset`. + :param value: Minimum root depth of the soil class, in the unit of ``Zr`` [cm]. + """ + path = os.path.join(os.path.dirname(config["TABLES"]["rootzone_depth"]), "Dpz_min.txt") + with open(path, "w", encoding="utf8") as f: + f.write(f"{_SOIL_CLASS} {value}\n") + return path diff --git a/tests/integration/test_modflow_coupling.py b/tests/integration/test_modflow_coupling.py new file mode 100644 index 0000000..392ca37 --- /dev/null +++ b/tests/integration/test_modflow_coupling.py @@ -0,0 +1,555 @@ +"""The MODFLOW groundwater module against the real ``mf2005`` on a 3x3 grid. + +One layer receives recharge and exchanges water with a river, a general head +boundary and a drain in every cell; with no lateral gradient each cell is a +closed balance whose head has an analytic solution (the cases of the +scientist's prototype, ported to the top-down configuration). The masked +variants leave the left column outside every layer, which exercises the +synthetic elevations of inactive columns. + +The three-layer cases check what one layer cannot show: the layer types +(LAYCON) reach MODFLOW on the right layers, and class lookups and numbers give +the same run as the equivalent maps. + +The full RUBEM runs couple the synthetic dataset to its three-layer section +(:func:`write_modflow_inputs`) through the validated configuration. +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pcraster as pcr +import pytest + +from rubem._deps import resolve_mf2005 +from rubem.configuration.model_configuration import ModelConfiguration +from rubem.configuration.modflow_configuration import ModflowSettings +from rubem.core import DynamicFrameworkWrapper +from rubem.hydrological_processes._modflow import ModflowGroundwater +from tests.helpers.synthetic import ( + CELL_SIZE, + MODFLOW_HEAD, + MODFLOW_TOP, + geotiff_series_name, + series_name, + write_grid_map, + write_modflow_inputs, + write_synthetic_dataset, +) + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif(resolve_mf2005() is None, reason="mf2005 is required"), +] + +GEOGRAPHIC_PIXEL = 0.0002945488721804391443 +CELL_AREA = 900.0 +DAYS = 30 +# 30 mm in 30 days is 0.001 m/day, 0.9 m3/day over a 900 m2 cell. +RECHARGE_MM = 30.0 +RECHARGE_FLOW = 0.9 +RIVER_STAGE = 50.0 +RIVER_CONDUCTANCE = 2.0 +PROBE_CELL = 2 # row 1, column 2: active in the masked variants too + + +class Grid: + """Rasters on a 3x3 clone, optionally missing on the left column.""" + + def __init__(self, directory, pixel_size, masked): + pcr.setclone(3, 3, pixel_size, 0, 3 * pixel_size) + self.directory = directory + self.mask = pcr.xcoordinate(pcr.spatial(pcr.boolean(1))) > pixel_size + self.masked = masked + + def field(self, value): + field = pcr.spatial(pcr.scalar(value)) + return pcr.ifthen(self.mask, field) if self.masked else field + + def map(self, name, value, nominal=False): + field = pcr.spatial(pcr.nominal(value) if nominal else pcr.scalar(value)) + if self.masked: + field = pcr.ifthen(self.mask, field) + path = str(self.directory / f"{name}.map") + pcr.report(field, path) + return path + + +def one_layer_section(grid, laytype, **packages): + return { + "enabled": True, + "top": grid.map("top", 100), + "layers": [ + { + "name": "aquifer", + "bottom": grid.map("bottom", 0), + "boundary": grid.map("boundary", 1, nominal=True), + "initial_head": grid.map("initial", 50), + "laytype": laytype, + "horizontal_conductivity": grid.map("kh", 1), + "vertical_conductivity": grid.map("kv", 1), + "specific_yield": grid.map("sy", 0.1), + } + ], + "solver": {"hclose": 1e-8, "rclose": 1e-8, "damp": 1}, + "river": { + "enabled": True, + "entries": [ + { + "layers": [1], + "stage": grid.map("stage", RIVER_STAGE), + "bottom": grid.map("bed", 10), + "conductance": grid.map("rivcond", RIVER_CONDUCTANCE), + } + ], + }, + **packages, + } + + +@pytest.fixture(name="directories") +def directories_fixture(tmp_path, monkeypatch): + """A run directory, and an empty working directory that must stay empty.""" + run_directory = tmp_path / "run" + working_directory = tmp_path / "cwd" + working_directory.mkdir() + monkeypatch.chdir(working_directory) + return run_directory, working_directory + + +def assert_files_only_in_run_directory(run_directory, working_directory): + assert (run_directory / "pcrmf.lst").is_file() + assert os.listdir(working_directory) == [] + + +@pytest.mark.parametrize("external_head, conductance", [(60, 3), (40, 3), (60, 0)]) +@pytest.mark.parametrize("masked", [False, True]) +@pytest.mark.parametrize("transient", [False, True]) +@pytest.mark.parametrize("pixel_size", [30.0, GEOGRAPHIC_PIXEL]) +def test_river_and_general_head_balance( + tmp_path, directories, external_head, conductance, masked, transient, pixel_size +): + """The metric recharge and storage balance holds whatever the clone units.""" + grid = Grid(tmp_path, pixel_size, masked) + section = one_layer_section( + grid, + laytype=1 if transient else 0, + dis={"steady_state": not transient, "nstp": 1}, + wetting={"enabled": transient, "map": grid.map("wet", -0.1), "layers": [1]}, + ghb={ + "enabled": True, + "entries": [ + { + "layers": [1], + "head": grid.map("external", external_head), + "conductance": grid.map("ghbcond", conductance), + } + ], + }, + ) + run_directory, working_directory = directories + model = ModflowGroundwater(ModflowSettings.model_validate(section), CELL_AREA, run_directory) + model.initialize(DAYS) + expected_head = 50.0 + for _ in range(2): + # Sy * area / duration is the implicit storage coefficient [m2/day]. + storage = 0.1 * CELL_AREA / DAYS if transient else 0.0 + expected_head = ( + conductance * external_head + + RIVER_CONDUCTANCE * RIVER_STAGE + + RECHARGE_FLOW + + storage * expected_head + ) / (conductance + RIVER_CONDUCTANCE + storage) + expected_river = RIVER_CONDUCTANCE * (RIVER_STAGE - expected_head) + result = model.run_step(grid.field(RECHARGE_MM), DAYS) + assert pcr.cellvalue(result.heads[1], PROBE_CELL)[0] == pytest.approx(expected_head) + river = pcr.cellvalue(result.net_river_leakage_m3_per_day, PROBE_CELL)[0] + assert river == pytest.approx(expected_river, abs=1e-4) + assert pcr.cellvalue(result.baseflow_mm, PROBE_CELL)[0] == pytest.approx( + max(-expected_river, 0) * DAYS * 1000 / CELL_AREA, abs=1e-4 * DAYS * 1000 / CELL_AREA + ) + if conductance: + ghb = pcr.cellvalue(model.mf.getGeneralHeadLeakage(1), PROBE_CELL)[0] + assert ghb == pytest.approx(conductance * (external_head - expected_head), abs=1e-4) + assert_files_only_in_run_directory(run_directory, working_directory) + + +@pytest.mark.parametrize("masked", [False, True]) +@pytest.mark.parametrize( + "elevation, conductance, enabled, output", + [ + (50, 4, True, True), + (56.18, 4, True, True), + (65, 4, True, True), + (50, 0, True, True), + (50, 4, False, True), + (50, 4, True, False), + ], +) +def test_drain_with_general_head_river_and_recharge( + tmp_path, directories, masked, elevation, conductance, enabled, output +): + grid = Grid(tmp_path, GEOGRAPHIC_PIXEL, masked) + section = one_layer_section( + grid, + laytype=0, + dis={"steady_state": True, "nstp": 1}, + ghb={ + "enabled": True, + "entries": [ + { + "layers": [1], + "head": grid.map("external", 60), + "conductance": grid.map("ghbcond", 3), + } + ], + }, + drain={ + "enabled": enabled, + "entries": [ + { + "layers": [1], + "elevation": grid.map("drnelev", elevation), + "conductance": grid.map("drncond", conductance), + } + ], + }, + output={"drain_flow": output}, + ) + run_directory, working_directory = directories + model = ModflowGroundwater(ModflowSettings.model_validate(section), CELL_AREA, run_directory) + model.initialize(DAYS) + # Without the drain: RCH 0.9 m3/day, GHB C=3 at 60 m, RIV C=2 at 50 m, so + # h = (3 * 60 + 2 * 50 + 0.9) / 5 = 56.18 m; the drain acts only below it. + effective = conductance if enabled and elevation < 56.18 else 0 + expected_head = (280.9 + effective * elevation) / (5 + effective) + expected_drain = -effective * max(expected_head - elevation, 0) + expected_river = RIVER_CONDUCTANCE * (RIVER_STAGE - expected_head) + for _ in range(2): + result = model.run_step(grid.field(RECHARGE_MM), DAYS) + assert pcr.cellvalue(result.heads[1], PROBE_CELL)[0] == pytest.approx( + expected_head, abs=1e-4 + ) + assert pcr.cellvalue(result.baseflow_mm, PROBE_CELL)[0] == pytest.approx( + max(-expected_river, 0) * DAYS * 1000 / CELL_AREA, abs=0.01 + ) + if enabled and output: + drain = pcr.cellvalue(result.drain_flow[1], PROBE_CELL)[0] + assert drain == pytest.approx(expected_drain, abs=1e-4) + else: + assert result.drain_flow == {} + assert_files_only_in_run_directory(run_directory, working_directory) + + +def bcf_layer_types(run_directory): + """The LAYCON of each layer in the BCF file, MODFLOW order (top layer first).""" + lines = (run_directory / "pcrmf.bc6").read_text(encoding="utf8").splitlines() + return [int(value) for value in lines[1].split()] + + +def test_the_layer_types_of_the_synthetic_dataset_reach_their_layers(tmp_path, directories): + """LAYCON 1 on the top layer and 2 below: MODFLOW refuses LAYCON 1 on any other layer.""" + config = write_synthetic_dataset(str(tmp_path / "dataset")) + settings = ModflowSettings.model_validate(write_modflow_inputs(config)) + run_directory, working_directory = directories + model = ModflowGroundwater(settings, CELL_SIZE * CELL_SIZE, run_directory) + model.initialize(DAYS) + for _ in range(2): + result = model.run_step(pcr.spatial(pcr.scalar(RECHARGE_MM)), DAYS) + assert sorted(result.heads) == [1, 2, 3] + for head in result.heads.values(): + assert np.isfinite(pcr.pcr2numpy(head, np.nan)).all() + assert bcf_layer_types(run_directory) == [1, 2, 2] + listing = (run_directory / "pcrmf.lst").read_text(encoding="utf8", errors="replace") + assert "LAYER TYPE 1 IS ONLY ALLOWED" not in listing + assert_files_only_in_run_directory(run_directory, working_directory) + + +ACTIVE = np.array([[False, True, True]] * 3) +RIVER = np.array([[False, False, True]] * 3) +CLASSES = np.array([[1, 1, 2], [2, 2, 1], [1, 2, 1]]) + + +def three_layer_section(directory, laytypes, calibrated): + """Three layers 20 m thick under a top at 60 m, the left column inactive. + + Layer ``n`` (top down) has the conductivity ``0.5 n`` in class 1 and ``2 n`` + in class 2. ``calibrated`` gives it as a class lookup, the storage and the + river conductance as numbers; otherwise every input is a map. The river + and the drain lie on layer 1, the general head boundary on layer 3, and + wetting applies to every LAYCON 1 or 3 layer. + """ + pcr.setclone(3, 3, 30, 0, 90) + directory.mkdir() + + def raster(name, values, nominal=False): + array = np.broadcast_to(np.asarray(values, dtype=float), (3, 3)).copy() + array[~ACTIVE] = np.nan + field = pcr.numpy2pcr(pcr.Scalar, array, np.nan) + if nominal: + field = pcr.nominal(field) + path = directory / f"{name}.map" + pcr.report(field, str(path)) + return str(path) + + layers = [] + for number, laytype in enumerate(laytypes, start=1): + table = directory / f"kh{number}.tbl" + table.write_text(f"1 {number * 0.5}\n2 {number * 2.0}\n", encoding="ascii") + kh = np.where(CLASSES == 1, number * 0.5, number * 2.0) + layers.append( + { + "name": f"layer{number}", + "bottom": raster(f"bottom{number}", 60 - 20 * number), + "boundary": raster(f"boundary{number}", 1, nominal=True), + "initial_head": raster(f"head{number}", 50), + "laytype": laytype, + "horizontal_conductivity": ( + {"map": raster(f"classes{number}", CLASSES, nominal=True), "table": str(table)} + if calibrated + else raster(f"kh{number}", kh) + ), + "vertical_conductivity": raster(f"kv{number}", 0.2), + "specific_storage": 0.0001 if calibrated else raster(f"ss{number}", 0.0001), + "specific_yield": 0.1 if calibrated else raster(f"sy{number}", 0.1), + } + ) + river = { + "layers": [1], + "stage": raster("stage", np.where(RIVER, 51, np.nan)), + "bottom": raster("bed", np.where(RIVER, 41, np.nan)), + "conductance": 2 if calibrated else raster("rivcond", np.where(RIVER, 2, 0)), + } + if calibrated: + river["mask"] = raster("rivers", RIVER) + return { + "enabled": True, + "top": raster("top", 60), + "layers": layers, + "dis": {"nstp": 2}, + "solver": {"hclose": 1e-6, "rclose": 1e-6, "damp": 1}, + "wetting": { + "enabled": True, + "map": raster("wet", 1), + "layers": [n for n, laytype in enumerate(laytypes, start=1) if laytype in (1, 3)], + }, + "river": {"enabled": True, "entries": [river]}, + "ghb": { + "enabled": True, + "entries": [ + {"layers": [3], "head": raster("ghbhead", 52), "conductance": raster("ghbcond", 1)} + ], + }, + "drain": { + "enabled": True, + "entries": [ + { + "layers": [1], + "elevation": raster("drnelev", 50.5), + "conductance": raster("drncond", 0.5), + } + ], + }, + "output": {"storage": True, "drain_flow": True}, + } + + +def run_three_layers(directory, laytypes, calibrated): + section = three_layer_section(directory, laytypes, calibrated) + run_directory = directory / "run" + model = ModflowGroundwater(ModflowSettings.model_validate(section), 900, run_directory) + model.initialize(30) + results = [] + for _ in range(2): + result = model.run_step(pcr.spatial(pcr.scalar(30)), 30) + assert sorted(result.heads) == sorted(result.storage) == [1, 2, 3] + assert sorted(result.drain_flow) == [1] + for item in [ + *result.heads.values(), + *result.storage.values(), + *result.drain_flow.values(), + result.baseflow_mm, + result.net_river_leakage_m3_per_day, + ]: + results.append(pcr.pcr2numpy(item, np.nan)[ACTIVE]) + assert bcf_layer_types(run_directory) == list(laytypes) + return results + + +@pytest.mark.parametrize("laytypes", [(1, 2, 0), (3, 3, 2)]) +def test_three_layer_constants_and_lookups_match_maps(tmp_path, directories, laytypes): + maps = run_three_layers(tmp_path / "maps", laytypes, calibrated=False) + calibrated = run_three_layers(tmp_path / "calibrated", laytypes, calibrated=True) + assert len(maps) == len(calibrated) + for expected, actual in zip(maps, calibrated, strict=True): + np.testing.assert_allclose(actual, expected, rtol=1e-6, atol=1e-6) + + +MONTH_DAYS = {1: 31, 2: 29} # January and February 2000 + + +def coupled_dataset(directory, **section_updates): + """The synthetic dataset coupled to MODFLOW, its river below the water table. + + The river stage is 5 m below the initial head, so the aquifer drains into + the river and the baseflow is not zero. + """ + config = write_synthetic_dataset(str(directory)) + section = write_modflow_inputs(config) + river = section["river"]["entries"][0] + river["stage"] = write_grid_map(Path(river["stage"]), MODFLOW_HEAD - 5.0) + river["bottom"] = write_grid_map(Path(river["bottom"]), MODFLOW_TOP - 18.0) + section.update(section_updates) + config["MODFLOW"] = section + return config + + +def run_rubem(config): + DynamicFrameworkWrapper.load(ModelConfiguration(config)).run() + + +def read_map(path): + return pcr.pcr2numpy(pcr.scalar(pcr.readmap(str(path))), np.nan) + + +def run_broken_solver_in_a_child(tmp_path, dis=None): + """Run the coupled dataset with a solver that cannot converge, in a child process. + + The child prints ``RAISED `` when the run raises ``RuntimeError`` + and ``RETURNED`` when it ends normally; a child the extension ends prints + neither. + + :param dis: The ``dis`` object of the section; ``None`` leaves the default. + """ + root = Path(__file__).resolve().parents[2] + working_directory = tmp_path / "cwd" + working_directory.mkdir() + script = ( + "import json, sys\n" + "from tests.integration.test_modflow_coupling import coupled_dataset, run_rubem\n" + "dis = json.loads(sys.argv[2])\n" + "config = coupled_dataset(sys.argv[1], solver={'mxiter': 1, 'iter1': 1," + " 'hclose': 1e-12, 'rclose': 1e-12}, **({'dis': dis} if dis else {}))\n" + "assert ('dis' in config['MODFLOW']) == bool(dis)\n" + "try:\n" + " run_rubem(config)\n" + "except RuntimeError as error:\n" + " print('RAISED', error)\n" + "else:\n" + " print('RETURNED')\n" + ) + environment = {**os.environ, "PYTHONPATH": os.pathsep.join([str(root), *sys.path])} + return subprocess.run( + [sys.executable, "-c", script, str(tmp_path / "dataset"), json.dumps(dis)], + cwd=working_directory, + env=environment, + capture_output=True, + text=True, + errors="replace", + timeout=600, + check=False, + ) + + +class TestCoupledRun: + def test_the_baseflow_is_the_river_leakage_and_the_heads_are_written( + self, tmp_path, directories, monkeypatch + ): + config = coupled_dataset(tmp_path / "dataset") + output = Path(config["DIRECTORIES"]["output"]) + run_directory, working_directory = directories + during = [] + run_step = ModflowGroundwater.run_step + + def observed_run_step(self, recharge_mm, days_in_period): + result = run_step(self, recharge_mm, days_in_period) + during.append( + ( + Path(self.run_directory), + (output / "modflow" / "pcrmf.lst").is_file(), + sorted(path.name for path in output.glob("pcrmf*")), + os.listdir(working_directory), + ) + ) + return result + + monkeypatch.setattr(ModflowGroundwater, "run_step", observed_run_step) + + run_rubem(config) + + assert during == [(output / "modflow", True, [], [])] * len(MONTH_DAYS) + assert not (output / "modflow").exists() + assert os.listdir(working_directory) == [] + for step, days in MONTH_DAYS.items(): + baseflow = read_map(output / series_name("bfw", step)) + leakage = read_map(output / series_name("mfaq2rv", step)) + assert leakage.max() > 0 + np.testing.assert_allclose( + baseflow, leakage * days * 1000.0 / (CELL_SIZE * CELL_SIZE), rtol=1e-5, atol=1e-6 + ) + for number in (1, 2, 3): + prefix = f"mfh{number}" + assert (output / series_name(prefix, step)).is_file(), (prefix, step) + assert (output / geotiff_series_name(prefix, step)).is_file(), (prefix, step) + assert np.isfinite(read_map(output / series_name(prefix, step))).all() + + def test_nothing_is_written_without_a_raster_format(self, tmp_path, directories): + config = coupled_dataset(tmp_path / "dataset") + config["RASTER_FILE_FORMAT"] = {"map_raster_series": False, "tiff_raster_series": False} + config["GENERATE_FILE"] = {key: False for key in config["GENERATE_FILE"]} + + run_rubem(config) + + assert os.listdir(config["DIRECTORIES"]["output"]) == [] + assert os.listdir(directories[1]) == [] + + def test_non_convergence_stops_the_run_and_keeps_the_listing(self, tmp_path, directories): + config = coupled_dataset( + tmp_path / "dataset", + dis={"nstp": 1}, + solver={"mxiter": 1, "iter1": 1, "hclose": 1e-12, "rclose": 1e-12}, + ) + + with pytest.raises(RuntimeError, match="did not converge in stress period 1"): + run_rubem(config) + + assert (Path(config["DIRECTORIES"]["output"]) / "modflow" / "pcrmf.lst").is_file() + + def test_non_convergence_with_the_default_time_steps_raises(self, tmp_path): + """The default ``dis`` (one time step per period) keeps the failure recoverable. + + The run happens in a child process because the opposite outcome, the + extension ending the process, would take the test runner with it. + """ + child = run_broken_solver_in_a_child(tmp_path) + + output = child.stdout + child.stderr + assert child.returncode == 0, output + assert "RAISED MODFLOW did not converge in stress period 1" in child.stdout + assert "MODFLOW failed to converge" in output + listing = tmp_path / "dataset" / "out" / "modflow" / "pcrmf.lst" + assert "STOPPING SIMULATION" in listing.read_text(encoding="utf8", errors="replace") + + def test_non_convergence_before_the_last_time_step_ends_the_process(self, tmp_path): + """Pins the documented limitation of ``dis.nstp`` above 1. + + MODFLOW saves the heads only at the last time step of the period and + stops at the first time step that fails to converge. With several time + steps nothing is saved in the first period, and the extension ends the + process when it reads the missing head file, before ``converged()`` + can be asked. The run therefore happens in a child process. + """ + child = run_broken_solver_in_a_child(tmp_path, dis={"nstp": 5}) + + output = child.stdout + child.stderr + assert child.returncode != 0, output + assert "RAISED" not in child.stdout + assert "RETURNED" not in child.stdout + assert "MODFLOW failed to converge" in output + assert "Can not open head value result file" in output + listing = tmp_path / "dataset" / "out" / "modflow" / "pcrmf.lst" + assert "STOPPING SIMULATION" in listing.read_text(encoding="utf8", errors="replace") diff --git a/tests/unit/calibration/test_modflow_parameters.py b/tests/unit/calibration/test_modflow_parameters.py new file mode 100644 index 0000000..d3bd698 --- /dev/null +++ b/tests/unit/calibration/test_modflow_parameters.py @@ -0,0 +1,419 @@ +"""The MODFLOW parameters a calibration may search: catalog, domain and document patching.""" + +import json +import pickle +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +from rubem.calibration.modflow_parameters import MODFLOW_PREFIX, ModflowCatalog, catalog +from rubem.calibration.parameters import decision_space +from rubem.configuration.modflow_configuration import ModflowSettings +from tests.helpers.config import REPO_ROOT + + +def write_table(path, text): + path.write_text(text, encoding="utf8") + return str(path) + + +def layer(name, laytype, **overrides): + """One layer whose rasters are paths that are never read by the catalog.""" + item = { + "name": name, + "bottom": f"/maps/{name}_bottom.map", + "initial_head": f"/maps/{name}_head.map", + "boundary": "/maps/bound.map", + "laytype": laytype, + "horizontal_conductivity": 1.0, + "vertical_conductivity": 0.1, + "specific_yield": 0.15, + "specific_storage": 1e-5, + } + item.update(overrides) + return item + + +def section(layers, rivers=None, steady_state=False): + """An enabled MODFLOW section with the given layers and river entries.""" + entries = rivers or [ + { + "layers": [1], + "stage": "/maps/stage.map", + "bottom": "/maps/riv_bottom.map", + "conductance": 0.387, + "mask": "/maps/riv_mask.map", + } + ] + return { + "enabled": True, + "top": "/maps/top.map", + "layers": layers, + "dis": {"steady_state": steady_state}, + "river": {"enabled": True, "entries": entries}, + } + + +@pytest.fixture +def kh_table(tmp_path): + return write_table(tmp_path / "kh1.tbl", "1 0.5\n2 0.1\n") + + +@pytest.fixture +def two_layers(kh_table): + """Layer 1 unconfined with kh classes, layer 2 convertible with a storage map.""" + return section( + [ + layer( + "upper", + 1, + horizontal_conductivity={"map": "/maps/kh_classes1.map", "table": kh_table}, + ), + layer("lower", 2, specific_storage="/maps/ss.map"), + ], + rivers=[ + { + "layers": [1], + "stage": "/maps/stage.map", + "bottom": "/maps/riv_bottom.map", + "conductance": 0.387, + "mask": "/maps/riv_mask.map", + }, + { + "layers": [2], + "stage": "/maps/stage.map", + "bottom": "/maps/riv_bottom.map", + "conductance": "/maps/riv_cond.map", + }, + ], + ) + + +def document_of(settings_section): + """A format 1.0 document reduced to what the patching touches.""" + return {"modflow": ModflowSettings.model_validate(settings_section).model_dump(mode="json")} + + +class TestCatalog: + @pytest.mark.unit + def test_the_numbers_and_the_table_rows_the_run_reads_are_named_top_down(self, two_layers): + found = catalog(ModflowSettings.model_validate(two_layers)) + + # LAYCON 1 reads only its specific yield; layer 2 has its specific + # storage as a map, which is not a number to search; the second river + # entry has a conductance map. + assert found.names == ( + "modflow.layers.1.specific_yield", + "modflow.layers.1.kh.1", + "modflow.layers.1.kh.2", + "modflow.layers.2.specific_yield", + "modflow.river.1.conductance", + ) + assert found.values == { + "modflow.layers.1.specific_yield": 0.15, + "modflow.layers.1.kh.1": 0.5, + "modflow.layers.1.kh.2": 0.1, + "modflow.layers.2.specific_yield": 0.15, + "modflow.river.1.conductance": 0.387, + } + assert all(name.startswith(MODFLOW_PREFIX) for name in found.names) + + @pytest.mark.unit + @pytest.mark.parametrize( + ("laytype", "expected"), + [ + (0, ["specific_storage"]), + (1, ["specific_yield"]), + (2, ["specific_yield", "specific_storage"]), + (13, ["specific_yield", "specific_storage"]), + ], + ) + def test_only_the_storage_the_layer_type_reads_is_named(self, laytype, expected): + found = catalog(ModflowSettings.model_validate(section([layer("only", laytype)]))) + + assert [ + name.rsplit(".", 1)[1] for name in found.names if name.startswith("modflow.layers.") + ] == expected + + @pytest.mark.unit + def test_a_steady_state_run_names_no_storage(self): + found = catalog( + ModflowSettings.model_validate(section([layer("only", 2)], steady_state=True)) + ) + + assert found.names == ("modflow.river.1.conductance",) + + @pytest.mark.unit + def test_numeric_class_keys_are_canonical_and_intervals_are_not_named(self, tmp_path): + table = write_table(tmp_path / "kh.tbl", "01 0.5\n2.0 0.1\n[3,5] 0.2\n1 9.0\n") + found = catalog( + ModflowSettings.model_validate( + section( + [ + layer( + "only", + 1, + horizontal_conductivity={"map": "/maps/classes.map", "table": table}, + ) + ] + ) + ) + ) + + # PCRaster uses the first row that matches a class, so the second row + # of class 1 is never read and the name carries the first one's value. + assert found.values["modflow.layers.1.kh.1"] == 0.5 + assert found.values["modflow.layers.1.kh.2"] == 0.1 + assert not [name for name in found.names if "[" in name] + # The rows themselves are kept as they are written. + assert found.tables[1] == [("01", 0.5), ("2.0", 0.1), ("[3,5]", 0.2), ("1", 9.0)] + + @pytest.mark.unit + @pytest.mark.parametrize("interval", ["[1,3]", "<1,>", "[2,2]"]) + def test_a_class_an_earlier_interval_row_covers_is_not_named(self, tmp_path, interval): + table = write_table(tmp_path / "kh.tbl", f"{interval} 5\n2 7\n") + settings = ModflowSettings.model_validate( + section( + [ + layer( + "only", + 1, + horizontal_conductivity={"map": "/maps/classes.map", "table": table}, + ) + ] + ) + ) + + found = catalog(settings) + + # PCRaster reads the interval row for class 2, so searching the second + # row would spend the budget on a value the run never sees. + assert "modflow.layers.1.kh.2" not in found.names + with pytest.raises(ValueError, match=r"modflow\.layers\.1\.specific_yield"): + decision_space(bounds={"modflow.layers.1.kh.2": (0.1, 1.0)}, modflow=found) + + @pytest.mark.unit + def test_a_class_after_an_interval_that_does_not_cover_it_is_named(self, tmp_path): + table = write_table(tmp_path / "kh.tbl", "[3,5] 5\n<0,2> 6\n2 7\n") + settings = ModflowSettings.model_validate( + section( + [ + layer( + "only", + 1, + horizontal_conductivity={"map": "/maps/classes.map", "table": table}, + ) + ] + ) + ) + + assert catalog(settings).values["modflow.layers.1.kh.2"] == 7.0 + + @pytest.mark.unit + def test_the_catalog_is_plain_data_that_crosses_to_a_worker(self, two_layers): + found = catalog(ModflowSettings.model_validate(two_layers)) + + assert isinstance(found, ModflowCatalog) + # The process pool pickles the context of every evaluation; the bytes + # here are the ones this test produced. + assert pickle.loads(pickle.dumps(found)) == found + + @pytest.mark.unit + def test_importing_the_module_imports_no_native_library(self): + completed = subprocess.run( + [ + sys.executable, + "-c", + "import sys\n" + "import rubem.calibration.modflow_parameters\n" + "import rubem.calibration.parameters\n" + "print(sorted(m for m in sys.modules " + "if m.split('.')[0] in ('pcraster', 'osgeo', 'scipy')))\n", + ], + capture_output=True, + text=True, + check=True, + cwd=REPO_ROOT, + ) + + assert completed.stdout.strip() == "[]" + + +class TestDomain: + @pytest.mark.unit + @pytest.mark.parametrize( + ("name", "value", "admissible"), + [ + ("modflow.layers.1.specific_yield", 1.0, True), + ("modflow.layers.1.specific_yield", 0.3, True), + ("modflow.layers.1.specific_yield", 0.0, False), + ("modflow.layers.1.specific_yield", 1.5, False), + ("modflow.layers.1.kh.1", 1e-9, True), + ("modflow.layers.1.kh.1", 0.0, False), + ("modflow.layers.1.kh.1", -1.0, False), + ("modflow.layers.1.kh.1", float("inf"), False), + ("modflow.layers.1.kh.1", float("nan"), False), + ("modflow.river.1.conductance", 50.0, True), + ("modflow.river.1.conductance", 0.0, False), + ("modflow.layers.2.specific_yield", 0.0, False), + ], + ) + def test_the_values_a_candidate_may_carry(self, two_layers, name, value, admissible): + found = catalog(ModflowSettings.model_validate(two_layers)) + + assert found.admits(name, value) is admissible + + @pytest.mark.unit + def test_the_domain_is_described_for_the_messages(self, two_layers): + found = catalog(ModflowSettings.model_validate(two_layers)) + + assert found.domain("modflow.layers.1.specific_yield") == "(0, 1]" + assert found.domain("modflow.layers.1.kh.2") == "(0, inf)" + + +class TestApply: + @pytest.mark.unit + def test_numbers_are_replaced_in_place(self, two_layers, tmp_path): + found = catalog(ModflowSettings.model_validate(two_layers)) + document = document_of(two_layers) + before = json.loads(json.dumps(document)) + + found.apply( + document, + { + "alpha": 4.5, + "modflow.layers.2.specific_yield": 0.2, + "modflow.river.1.conductance": 1, + }, + tmp_path / "evaluation", + ) + + assert document["modflow"]["layers"][1]["specific_yield"] == 0.2 + conductance = document["modflow"]["river"]["entries"][0]["conductance"] + assert conductance == 1.0 + assert isinstance(conductance, float) + # Nothing else moved: the kh table is only rewritten when a class is named. + before["modflow"]["layers"][1]["specific_yield"] = 0.2 + before["modflow"]["river"]["entries"][0]["conductance"] = 1.0 + assert document == before + assert not (tmp_path / "evaluation").exists() + ModflowSettings.model_validate(document["modflow"]) + + @pytest.mark.unit + def test_a_named_class_rewrites_the_table_of_its_layer(self, two_layers, kh_table, tmp_path): + found = catalog(ModflowSettings.model_validate(two_layers)) + document = document_of(two_layers) + table_dir = tmp_path / "evaluation" + table_dir.mkdir() + + found.apply(document, {"modflow.layers.1.kh.2": 0.25}, table_dir) + + written = table_dir / "kh_layer1.tbl" + lookup = document["modflow"]["layers"][0]["horizontal_conductivity"] + assert Path(lookup["table"]) == written + assert lookup["map"] == "/maps/kh_classes1.map" + assert written.read_text(encoding="utf8").split() == ["1", "0.5", "2", "0.25"] + # The configured table is never touched. + assert (tmp_path / "kh1.tbl").read_text(encoding="utf8") == "1 0.5\n2 0.1\n" + + @pytest.mark.unit + def test_the_rows_that_are_not_named_are_kept_verbatim(self, tmp_path): + table = write_table(tmp_path / "kh.tbl", "01 0.5\n[3,5] 0.2\n1 9.0\n") + settings = section( + [layer("only", 1, horizontal_conductivity={"map": "/maps/c.map", "table": table})] + ) + found = catalog(ModflowSettings.model_validate(settings)) + document = document_of(settings) + + found.apply(document, {"modflow.layers.1.kh.1": 0.75}, tmp_path) + + rows = [ + line.split() for line in (tmp_path / "kh_layer1.tbl").read_text("utf8").splitlines() + ] + assert rows == [["01", "0.75"], ["[3,5]", "0.2"], ["1", "9"]] + + @pytest.mark.unit + def test_the_table_name_can_be_chosen(self, two_layers, tmp_path): + found = catalog(ModflowSettings.model_validate(two_layers)) + document = document_of(two_layers) + + found.apply( + document, {"modflow.layers.1.kh.1": 0.3}, tmp_path, table_prefix="basin-calibrated-kh" + ) + + assert (tmp_path / "basin-calibrated-kh1.tbl").is_file() + assert not (tmp_path / "kh_layer1.tbl").exists() + + @pytest.mark.unit + def test_without_a_modflow_name_the_document_is_left_alone(self, two_layers, tmp_path): + found = catalog(ModflowSettings.model_validate(two_layers)) + document = document_of(two_layers) + before = json.loads(json.dumps(document)) + + found.apply(document, {"alpha": 4.5, "w_3": 0.1}, tmp_path / "nothing") + + assert document == before + assert not (tmp_path / "nothing").exists() + + @pytest.mark.unit + def test_a_name_the_catalog_does_not_hold_is_refused(self, two_layers, tmp_path): + found = catalog(ModflowSettings.model_validate(two_layers)) + + with pytest.raises(ValueError, match=r"modflow\.layers\.2\.specific_storage"): + found.apply( + document_of(two_layers), {"modflow.layers.2.specific_storage": 1e-4}, tmp_path + ) + + @pytest.mark.unit + def test_pcraster_reads_the_rewritten_table(self, tmp_path): + import pcraster as pcr + + from tests.helpers.synthetic import COLS, ROWS, write_grid_map + + classes = write_grid_map( + tmp_path / "classes.map", [1] * COLS + [2] * (ROWS - 1) * COLS, True + ) + table = write_table(tmp_path / "kh.tbl", "1 0.5\n2 0.1\n") + settings = section( + [layer("only", 1, horizontal_conductivity={"map": classes, "table": table})] + ) + found = catalog(ModflowSettings.model_validate(settings)) + document = document_of(settings) + out = tmp_path / "out" + out.mkdir() + + # A value whose shortest repr is in scientific notation. + found.apply(document, {"modflow.layers.1.kh.2": 1e-7}, out) + + rewritten = document["modflow"]["layers"][0]["horizontal_conductivity"]["table"] + values = pcr.pcr2numpy(pcr.lookupscalar(rewritten, pcr.readmap(classes)), np.nan) + assert values[0].tolist() == pytest.approx([0.5] * COLS) + assert values[1:].ravel().tolist() == pytest.approx([1e-7] * (ROWS - 1) * COLS) + + @pytest.mark.unit + def test_pcraster_reads_a_class_rewritten_before_an_interval_that_covers_it(self, tmp_path): + import pcraster as pcr + + from tests.helpers.synthetic import COLS, ROWS, write_grid_map + + classes = write_grid_map( + tmp_path / "classes.map", [1] * COLS + [2] * (ROWS - 1) * COLS, True + ) + table = write_table(tmp_path / "kh.tbl", "2 7\n[1,3] 5\n") + settings = section( + [layer("only", 1, horizontal_conductivity={"map": classes, "table": table})] + ) + found = catalog(ModflowSettings.model_validate(settings)) + document = document_of(settings) + out = tmp_path / "out" + out.mkdir() + + found.apply(document, {"modflow.layers.1.kh.2": 0.9}, out) + + rewritten = document["modflow"]["layers"][0]["horizontal_conductivity"]["table"] + values = pcr.pcr2numpy(pcr.lookupscalar(rewritten, pcr.readmap(classes)), np.nan) + assert values[0].tolist() == pytest.approx([5.0] * COLS) + assert values[1:].ravel().tolist() == pytest.approx([0.9] * (ROWS - 1) * COLS) diff --git a/tests/unit/calibration/test_parameters.py b/tests/unit/calibration/test_parameters.py index a2c209a..bebd71e 100644 --- a/tests/unit/calibration/test_parameters.py +++ b/tests/unit/calibration/test_parameters.py @@ -3,6 +3,7 @@ import numpy as np import pytest +from rubem.calibration.modflow_parameters import ModflowCatalog from rubem.calibration.parameters import ( CALIBRATION_PARAMETERS, DERIVED_PARAMETER, @@ -453,3 +454,160 @@ def test_a_bound_of_the_derived_or_of_an_unknown_parameter_is_refused(self): def test_a_bound_of_a_fixed_parameter_has_nowhere_to_apply(self): with pytest.raises(ValueError, match="nowhere to apply"): decision_space(fixed={"rcd": 3.0}, bounds={"rcd": (2.0, 5.0)}) + + +SPECIFIC_YIELD = "modflow.layers.1.specific_yield" +KH = "modflow.layers.1.kh.2" +CONDUCTANCE = "modflow.river.1.conductance" + + +@pytest.fixture +def catalog(): + """The MODFLOW parameters of a configuration, in the order of its section.""" + return ModflowCatalog( + values={SPECIFIC_YIELD: 0.15, "modflow.layers.1.kh.1": 0.5, KH: 0.1, CONDUCTANCE: 0.387}, + tables={1: [("1", 0.5), ("2", 0.1)]}, + ) + + +class TestModflowParameters: + @pytest.mark.unit + def test_without_a_modflow_name_the_space_is_the_one_of_today(self, catalog): + assert decision_space(modflow=catalog) == decision_space() + assert decision_space( + fixed={"x": 0.0}, bounds={"rcd": (2.0, 5.0)}, modflow=catalog + ) == decision_space(fixed={"x": 0.0}, bounds={"rcd": (2.0, 5.0)}) + assert decision_space().modflow_names == () + + @pytest.mark.unit + def test_a_bounded_name_is_searched_after_the_eight_parameters(self, catalog): + space = decision_space(bounds={SPECIFIC_YIELD: (0.05, 0.3)}, modflow=catalog) + + assert space.free_names == (*FREE_PARAMETERS, SPECIFIC_YIELD) + assert space.bounds[-1] == (0.05, 0.3) + assert space.dimension == 9 + assert space.modflow_names == (SPECIFIC_YIELD,) + + @pytest.mark.unit + def test_the_names_follow_the_catalog_whatever_order_they_are_given_in(self, catalog): + space = decision_space( + bounds={CONDUCTANCE: (0.1, 0.5), KH: (0.05, 0.5), SPECIFIC_YIELD: (0.05, 0.3)}, + modflow=catalog, + ) + + assert space.free_names[8:] == (SPECIFIC_YIELD, KH, CONDUCTANCE) + assert space.bounds[8:] == ((0.05, 0.3), (0.05, 0.5), (0.1, 0.5)) + + @pytest.mark.unit + def test_a_candidate_carries_the_nine_parameters_and_the_modflow_ones(self, catalog): + space = decision_space( + fixed={CONDUCTANCE: 0.2}, bounds={SPECIFIC_YIELD: (0.05, 0.3)}, modflow=catalog + ) + vector = np.append(admissible_vector(), 0.25) + + parameters = space.to_parameters(vector) + + assert list(parameters) == [*CALIBRATION_PARAMETERS, SPECIFIC_YIELD, CONDUCTANCE] + assert parameters[SPECIFIC_YIELD] == 0.25 + assert parameters[CONDUCTANCE] == 0.2 + assert parameters["w_3"] == pytest.approx(1.0 - 0.666) + assert space.fixed == {CONDUCTANCE: 0.2} + assert space.modflow_names == (SPECIFIC_YIELD, CONDUCTANCE) + + @pytest.mark.unit + def test_the_starting_point_reads_the_modflow_values(self, catalog): + space = decision_space(bounds={KH: (0.05, 0.5)}, modflow=catalog) + parameters = vector_to_parameters(admissible_vector()) + + vector = space.from_parameters({**parameters, **catalog.values}) + + assert vector.tolist() == pytest.approx([*admissible_vector().tolist(), 0.1]) + + @pytest.mark.unit + def test_a_modflow_value_outside_its_bound_is_not_admissible(self, catalog): + space = decision_space(bounds={KH: (0.05, 0.5)}, modflow=catalog) + + assert space.is_admissible(np.append(admissible_vector(), 0.5)) + assert not space.is_admissible(np.append(admissible_vector(), 0.6)) + assert not space.is_admissible(np.append(admissible_vector(), np.nan)) + + @pytest.mark.unit + def test_the_weights_keep_their_positions_in_the_constraint(self, catalog): + pytest.importorskip("scipy.optimize") + space = decision_space(bounds={KH: (0.05, 0.5)}, modflow=catalog) + + coefficients = np.asarray(space.weights_constraint().A) + + assert coefficients.tolist() == [[0, 0, 1, 1, 0, 0, 0, 0, 0]] + + @pytest.mark.unit + def test_the_eight_parameters_may_all_be_fixed_when_a_modflow_one_is_searched(self, catalog): + every = {name: minimum for name, (minimum, _) in SETTINGS_BOUNDS.items()} + every.pop(DERIVED_PARAMETER) + every["w_1"], every["w_2"] = 0.5, 0.5 + + space = decision_space(fixed=every, bounds={KH: (0.05, 0.5)}, modflow=catalog) + + assert space.free_names == (KH,) + assert space.weights_constraint() is None + + @pytest.mark.unit + def test_a_name_the_catalog_does_not_hold_names_the_known_ones(self, catalog): + with pytest.raises(ValueError, match="modflow.layers.2.specific_yield") as failure: + decision_space(bounds={"modflow.layers.2.specific_yield": (0.1, 0.2)}, modflow=catalog) + + for name in catalog.names: + assert name in str(failure.value) + + @pytest.mark.unit + @pytest.mark.parametrize("argument", ["fixed", "bounds"]) + def test_a_modflow_name_without_modflow_is_refused(self, argument): + value = 0.2 if argument == "fixed" else (0.1, 0.3) + + with pytest.raises(ValueError, match="does not enable MODFLOW"): + decision_space(**{argument: {SPECIFIC_YIELD: value}}) + + @pytest.mark.unit + @pytest.mark.parametrize( + ("name", "override"), + [ + (KH, (0.0, 0.5)), + (KH, (-1.0, 0.5)), + (KH, (0.5, 0.1)), + (KH, (0.3, 0.3)), + (KH, (0.1, float("inf"))), + (KH, (float("nan"), 0.5)), + (CONDUCTANCE, (0.0, 1.0)), + (SPECIFIC_YIELD, (0.0, 0.3)), + (SPECIFIC_YIELD, (0.1, 1.2)), + ], + ) + def test_a_bound_outside_the_domain_of_the_parameter_is_refused(self, catalog, name, override): + with pytest.raises(ValueError, match=f"bound .* of '{name}'"): + decision_space(bounds={name: override}, modflow=catalog) + + @pytest.mark.unit + def test_a_specific_yield_may_be_searched_up_to_one(self, catalog): + space = decision_space(bounds={SPECIFIC_YIELD: (0.5, 1.0)}, modflow=catalog) + + assert space.bounds[-1] == (0.5, 1.0) + + @pytest.mark.unit + @pytest.mark.parametrize("override", [0.3, (0.3,), (0.1, 0.2, 0.3)]) + def test_a_modflow_bound_that_is_not_a_pair_is_refused(self, catalog, override): + with pytest.raises(ValueError, match=r"must be a \(minimum, maximum\) pair"): + decision_space(bounds={KH: override}, modflow=catalog) + + @pytest.mark.unit + @pytest.mark.parametrize( + ("name", "value"), + [(KH, 0.0), (CONDUCTANCE, -1.0), (SPECIFIC_YIELD, 1.5), (KH, float("nan"))], + ) + def test_a_fixed_value_outside_the_domain_is_refused(self, catalog, name, value): + with pytest.raises(ValueError, match=f"'{name}' cannot be fixed"): + decision_space(fixed={name: value}, modflow=catalog) + + @pytest.mark.unit + def test_a_modflow_name_both_fixed_and_bounded_is_refused(self, catalog): + with pytest.raises(ValueError, match="nowhere to apply"): + decision_space(fixed={KH: 0.2}, bounds={KH: (0.1, 0.3)}, modflow=catalog) diff --git a/tests/unit/calibration/test_runner.py b/tests/unit/calibration/test_runner.py index b5f88f2..80b06b9 100644 --- a/tests/unit/calibration/test_runner.py +++ b/tests/unit/calibration/test_runner.py @@ -45,7 +45,13 @@ ModelConfigurationFileV1, ) from tests.helpers.config import REPO_ROOT -from tests.helpers.synthetic import write_synthetic_dataset +from tests.helpers.synthetic import ( + MODFLOW_HEAD, + MODFLOW_KH_TABLE, + write_grid_map, + write_modflow_inputs, + write_synthetic_dataset, +) # SciPy is the optional ``rubem[calibration]`` extra, and an environment # without it (the frozen one of the byte-exact job, for instance) still @@ -1109,13 +1115,14 @@ def test_the_selected_stations_are_logged_before_the_search( assert "the station(s) 1" in caplog.text -def _record_then_raise(error): +def _record_then_raise(error, modflow=None): """A search that records one evaluation and then fails, without a worker. The records are what an interrupted calibration has to leave behind, and the failure has to happen where the real one does, inside the search. The evaluations directory is read out of the objective the search was handed, - which is where the parent put it. + which is where the parent put it. ``modflow`` adds MODFLOW values to the + parameters of the record. """ def stub(function, search_bounds, **kwargs): @@ -1124,7 +1131,7 @@ def stub(function, search_bounds, **kwargs): "id": "deadbeef", "pid": 4321, "started_at": "2000-01-01T00:00:00+00:00", - "parameters": dict.fromkeys(CALIBRATION_PARAMETERS, 0.5), + "parameters": {**dict.fromkeys(CALIBRATION_PARAMETERS, 0.5), **(modflow or {})}, "nse": 0.25, "station_nse": {"1": 0.25}, "station_metrics": {}, @@ -1191,6 +1198,14 @@ def test_a_worker_killed_by_the_system_is_reported_with_the_advice(self, dataset assert isinstance(failure.value.__cause__, BrokenProcessPool) assert read_evaluations(dataset.run_dir / "evaluations.csv") + @pytest.mark.unit + @pytest.mark.parametrize("function", ["_consolidate", "_write_evaluations_csv"]) + def test_the_columns_of_the_table_are_always_given(self, function): + # A default would silently fall back to the nine parameters and drop the + # MODFLOW columns of the one path that forgot to pass them. + parameter = inspect.signature(getattr(runner, function)).parameters["columns"] + assert parameter.default is inspect.Parameter.empty + class TestProgressReporting: @pytest.mark.unit @@ -1269,3 +1284,451 @@ def test_a_selection_that_names_one_unknown_station_warns_and_keeps_the_rest( dataset.calibrate(stations=("1", "7")) assert "does not have (7)" in caplog.text + + +SPECIFIC_YIELD = "modflow.layers.1.specific_yield" +KH = "modflow.layers.1.kh.2" + +needs_mf2005 = pytest.mark.skipif( + _deps.resolve_mf2005() is None, reason="the MODFLOW-2005 executable is not installed" +) + + +class ModflowDataset(Dataset): + """The synthetic dataset coupled to MODFLOW, and a series another candidate wrote. + + The configuration under calibration has the specific yield 0.15 and the + class 2 conductivity 0.1 of :func:`write_modflow_inputs`; the observed + series is what the same configuration writes with ``truth`` instead, so a + search that finds the observations has to move the MODFLOW values. + + The river stage lies below the initial head, so the aquifer drains into the + river and the baseflow, hence ``arn``, depends on the specific yield (with + the stage of :func:`write_modflow_inputs` above the heads the river only + loses water and the baseflow is zero whatever the MODFLOW values). The + default time discretization, one time step per stress period, is kept: a + worker whose MODFLOW run fails before the last time step of a period ends + its process, and the search could not tell that from a bug of the + calibration. + + :param dis: The ``dis`` object of the section, when not the default. + """ + + def __init__(self, tmp_path, truth=None, dis=None): + self.config = write_synthetic_dataset(str(tmp_path), timesteps=TIMESTEPS) + section = write_modflow_inputs(self.config) + if dis is not None: + section["dis"] = dis + river = section["river"]["entries"][0] + directory = Path(section["top"]).parent + river["stage"] = write_grid_map(directory / "drained_stage.map", MODFLOW_HEAD - 5.0) + river["bottom"] = write_grid_map(directory / "drained_bottom.map", MODFLOW_HEAD - 8.0) + self.config["MODFLOW"] = section + self.config_file = tmp_path / "config.json" + self.config_file.write_text(json.dumps(self.config), encoding="utf8") + self.truth = truth or {} + observed_config = json.loads(json.dumps(self.config)) + layer = observed_config["MODFLOW"]["layers"][0] + if SPECIFIC_YIELD in self.truth: + layer["specific_yield"] = self.truth[SPECIFIC_YIELD] + if KH in self.truth: + table = tmp_path / "kh_truth.tbl" + table.write_text(f"1 {MODFLOW_KH_TABLE[1]}\n2 {self.truth[KH]}\n", encoding="utf8") + layer["horizontal_conductivity"]["table"] = str(table) + written = Model.from_config(observed_config).run().time_series["arn"][0] + self.observed = tmp_path / "observed.csv" + shutil.copyfile(written, self.observed) + self.run_dir = tmp_path / "calibration" + self.temp_dir = tmp_path / "temp" + self.parameters = ModelConfiguration( + self.config, validate_input=False + ).calibration_parameters.model_dump() + + def tmp_run_dir(self, name): + """A fresh run directory, for the tests that share one dataset.""" + return self.run_dir.parent / f"calibration-{name}" + + def init(self, columns): + """``INIT`` widened by one column per MODFLOW name, with one member at the truth. + + The second member is the configuration's own eight free parameters with + the MODFLOW values of the observations, so the global optimum is in the + initial population and the search is deterministic. + + :param columns: The five values of each MODFLOW name, in the order of + the decision vector. + """ + rows = [ + [*row, *(values[member] for values in columns.values())] + for member, row in enumerate(INIT.tolist()) + ] + rows[1] = [ + *parameters_to_vector(self.parameters).tolist(), + *(self.truth.get(name, values[1]) for name, values in columns.items()), + ] + return np.asarray(rows, dtype=np.float64) + + +@pytest.fixture(scope="class") +def calibrated_modflow(tmp_path_factory): + """One search of the specific yield of layer 1, with one class conductivity pinned. + + The conductivity of a class barely moves ``arn`` on this grid (the sixth + significant digit), so it is fixed at the value of the observations rather + than searched; the fixed value still goes through every evaluation and into + the calibrated configuration. + """ + data = ModflowDataset( + tmp_path_factory.mktemp("calibration-modflow"), truth={SPECIFIC_YIELD: 0.25, KH: 0.2} + ) + result = data.calibrate( + bounds={SPECIFIC_YIELD: (0.05, 0.3)}, + fixed={KH: 0.2}, + init=data.init({SPECIFIC_YIELD: (0.1, 0.25, 0.2, 0.3, 0.05)}), + ) + return data, result + + +@needs_mf2005 +class TestModflowCalibration: + @pytest.mark.unit + def test_the_modflow_values_of_the_observations_are_found(self, calibrated_modflow): + data, result = calibrated_modflow + + assert result.best_nse == pytest.approx(1.0, abs=1e-9) + assert result.best_parameters == pytest.approx( + {**data.parameters, SPECIFIC_YIELD: 0.25, KH: 0.2}, abs=1e-12 + ) + assert list(result.best_parameters) == [*CALIBRATION_PARAMETERS, SPECIFIC_YIELD, KH] + + @pytest.mark.unit + def test_the_modflow_names_are_columns_of_the_table(self, calibrated_modflow): + _, result = calibrated_modflow + rows = read_evaluations(result.evaluations_csv) + columns = list(EVALUATION_COLUMNS) + columns[columns.index("x") + 1 : columns.index("x") + 1] = [SPECIFIC_YIELD, KH] + + assert list(rows[0]) == columns + assert [row["error"] for row in rows if row["error"]] == [] + assert all(row[SPECIFIC_YIELD] for row in rows) + assert len({row[SPECIFIC_YIELD] for row in rows}) > 1 + assert {row[KH] for row in rows} == {"0.2"} + + @pytest.mark.unit + def test_the_series_of_the_best_candidate_is_the_one_of_its_modflow_values( + self, calibrated_modflow + ): + _, result = calibrated_modflow + + # The best candidate reproduces the observations only when it is run + # again with its own MODFLOW values, not the ones of the configuration. + rows = read_station_table(result.best_series) + assert rows + for row in rows: + for station in ("1", "2"): + assert float(row[f"simulated_{station}"]) == pytest.approx( + float(row[f"observed_{station}"]), rel=1e-9, abs=1e-12 + ) + + @pytest.mark.unit + def test_the_summary_lists_the_modflow_parameters(self, calibrated_modflow): + _, result = calibrated_modflow + summary = json.loads(result.result_json.read_text(encoding="utf-8")) + + assert summary["best_parameters"][SPECIFIC_YIELD] == pytest.approx(0.25) + assert summary["best_parameters"][KH] == pytest.approx(0.2) + assert summary["settings"]["bounds"][SPECIFIC_YIELD] == [0.05, 0.3] + assert summary["settings"]["fixed"] == {KH: 0.2} + + @pytest.mark.unit + def test_the_calibrated_configuration_carries_the_modflow_values(self, calibrated_modflow): + data, result = calibrated_modflow + table = result.run_dir / "config-calibrated-kh1.tbl" + + document = json.loads(result.calibrated_config.read_text(encoding="utf-8")) + layer = document["MODFLOW"]["layers"][0] + + assert layer["specific_yield"] == pytest.approx(0.25) + assert Path(layer["horizontal_conductivity"]["table"]) == table + assert table.read_text(encoding="utf8").split() == ["1", "0.5", "2", "0.2"] + # The configured table is left as it was. + configured = Path(data.config["MODFLOW"]["layers"][0]["horizontal_conductivity"]["table"]) + assert configured.read_text(encoding="utf8").split() == ["1", "0.5", "2", "0.1"] + configuration = ModelConfiguration(result.calibrated_config, validate_input=False) + assert configuration.modflow.layers[0].specific_yield == pytest.approx(0.25) + + @pytest.mark.unit + def test_no_modflow_directory_or_table_is_left_in_the_temporary_directory( + self, calibrated_modflow + ): + data, _ = calibrated_modflow + + assert list(data.temp_dir.iterdir()) == [] + + +@pytest.fixture(scope="class") +def calibrated_modflow_without_names(tmp_path_factory): + """A coupled configuration calibrated on the nine parameters only.""" + data = ModflowDataset(tmp_path_factory.mktemp("calibration-modflow-nine")) + return data, data.calibrate() + + +@needs_mf2005 +class TestModflowConfigurationWithoutModflowNames: + @pytest.mark.unit + def test_the_table_has_the_columns_of_today(self, calibrated_modflow_without_names): + _, result = calibrated_modflow_without_names + rows = read_evaluations(result.evaluations_csv) + + assert list(rows[0]) == list(EVALUATION_COLUMNS) + assert [row["error"] for row in rows if row["error"]] == [] + + @pytest.mark.unit + def test_every_evaluation_runs_the_coupled_model(self, calibrated_modflow_without_names): + data, result = calibrated_modflow_without_names + + # The observations are the coupled run of the configuration, so only a + # coupled evaluation of the configuration reproduces them. + assert result.best_nse == pytest.approx(1.0, abs=1e-9) + assert list(result.best_parameters) == list(CALIBRATION_PARAMETERS) + document = json.loads(result.calibrated_config.read_text(encoding="utf-8")) + assert document["MODFLOW"] == json.loads( + json.dumps(ModelConfiguration(data.config_file).modflow.model_dump(mode="json")) + ) + assert not list(result.run_dir.glob("*.tbl")) + + +@pytest.fixture(scope="class") +def modflow_dataset(tmp_path_factory): + """A coupled dataset for the refusals, which never start a search.""" + return ModflowDataset(tmp_path_factory.mktemp("modflow-refusals")) + + +@needs_mf2005 +class TestModflowRefusals: + @pytest.mark.unit + @pytest.mark.parametrize( + ("bound", "match"), + [ + ({KH: (0.0, 0.5)}, "not a range inside"), + ({"modflow.layers.1.specific_storage": (1e-6, 1e-4)}, "calibratable MODFLOW"), + ({"modflow.layers.1.kh.9": (0.1, 0.2)}, "modflow.layers.1.kh.1"), + ], + ) + def test_a_modflow_bound_the_configuration_cannot_take_is_refused_before_the_search( + self, modflow_dataset, no_pool, bound, match + ): + with pytest.raises(ValueError, match=match): + calibrate( + modflow_dataset.config_file, + modflow_dataset.observed, + modflow_dataset.run_dir, + modflow_dataset.settings(bounds=bound), + ) + + assert not modflow_dataset.run_dir.exists() + assert no_pool == [] + + @pytest.mark.unit + def test_the_search_starts_from_the_modflow_values_of_the_configuration( + self, modflow_dataset, broken_search + ): + init = modflow_dataset.init({KH: (0.1, 0.2, 0.3, 0.4, 0.5)}) + with pytest.raises(RuntimeError, match="the search broke"): + calibrate( + modflow_dataset.config_file, + modflow_dataset.observed, + modflow_dataset.tmp_run_dir("start"), + modflow_dataset.settings(bounds={KH: (0.05, 0.5)}, init=init), + ) + + assert broken_search["bounds"][-1] == (0.05, 0.5) + assert len(broken_search["bounds"]) == 9 + assert broken_search["x0"][-1] == pytest.approx(MODFLOW_KH_TABLE[2]) + + @pytest.mark.unit + def test_the_workers_find_mf2005_on_the_path_they_inherit( + self, modflow_dataset, broken_search, monkeypatch + ): + order = [] + monkeypatch.setattr(_deps, "ensure_mf2005_on_path", lambda: order.append("path")) + built = runner._pool + + def pool(workers): + order.append("pool") + return built(workers) + + monkeypatch.setattr(runner, "_pool", pool) + + with pytest.raises(RuntimeError, match="the search broke"): + calibrate( + modflow_dataset.config_file, + modflow_dataset.observed, + modflow_dataset.tmp_run_dir("path"), + modflow_dataset.settings(), + ) + + assert order == ["path", "pool"] + + @pytest.mark.unit + def test_an_unreadable_conductivity_table_stops_the_calibration( + self, tmp_path, no_pool, caplog + ): + data = ModflowDataset(tmp_path) + table = Path(data.config["MODFLOW"]["layers"][0]["horizontal_conductivity"]["table"]) + table.write_text("1 0.5\n2 fast\n", encoding="utf8") + + with caplog.at_level(logging.CRITICAL): + with pytest.raises(CalibrationError, match="kh1.tbl"): + data.calibrate(allow_blocking_problems=True) + + assert no_pool == [] + + @pytest.mark.unit + def test_a_dead_worker_of_a_coupled_run_names_the_modflow_cause( + self, modflow_dataset, monkeypatch + ): + monkeypatch.setattr( + scipy_optimize, + "differential_evolution", + _record_then_raise(BrokenProcessPool("A process in the process pool was terminated")), + ) + + with pytest.raises(CalibrationError, match="memory") as failure: + calibrate( + modflow_dataset.config_file, + modflow_dataset.observed, + modflow_dataset.tmp_run_dir("dead"), + modflow_dataset.settings(), + ) + + assert "dis.nstp" in str(failure.value) + assert "converge" in str(failure.value) + + @pytest.mark.unit + @pytest.mark.parametrize( + ("error", "raised"), + [ + (BrokenProcessPool("A process in the process pool was terminated"), CalibrationError), + (KeyboardInterrupt(), KeyboardInterrupt), + ], + ids=["dead-worker", "interrupted"], + ) + def test_the_table_of_an_ended_search_keeps_the_modflow_columns( + self, modflow_dataset, monkeypatch, error, raised + ): + monkeypatch.setattr( + scipy_optimize, + "differential_evolution", + _record_then_raise(error, modflow={SPECIFIC_YIELD: 0.2}), + ) + run_dir = modflow_dataset.tmp_run_dir(f"ended-{type(error).__name__}") + + with pytest.raises(raised): + calibrate( + modflow_dataset.config_file, + modflow_dataset.observed, + run_dir, + modflow_dataset.settings( + bounds={SPECIFIC_YIELD: (0.05, 0.3)}, + init=modflow_dataset.init({SPECIFIC_YIELD: (0.1, 0.25, 0.2, 0.3, 0.05)}), + ), + ) + + # The table the message points to carries what the evaluations searched. + with (run_dir / "evaluations.csv").open(encoding="utf-8", newline="") as table: + header = next(csv.reader(table)) + position = EVALUATION_COLUMNS.index("x") + 1 + assert header == [ + *EVALUATION_COLUMNS[:position], + SPECIFIC_YIELD, + *EVALUATION_COLUMNS[position:], + ] + rows = read_evaluations(run_dir / "evaluations.csv") + assert [row[SPECIFIC_YIELD] for row in rows] == [repr(0.2)] + + +@needs_mf2005 +class TestModflowTimeSteps: + @pytest.mark.unit + def test_several_time_steps_per_period_are_warned_about_before_the_search( + self, tmp_path, broken_search, caplog + ): + data = ModflowDataset(tmp_path, dis={"nstp": 5}) + + with caplog.at_level(logging.WARNING, logger="rubem.calibration.runner"): + with pytest.raises(RuntimeError, match="the search broke"): + data.calibrate() + + warnings = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "dis.nstp" in record.getMessage() + ] + assert len(warnings) == 1 + assert "5" in warnings[0] + assert "worker" in warnings[0] and "converge" in warnings[0] + + @pytest.mark.unit + def test_one_time_step_per_period_is_not_warned_about( + self, modflow_dataset, broken_search, caplog + ): + with caplog.at_level(logging.WARNING, logger="rubem.calibration.runner"): + with pytest.raises(RuntimeError, match="the search broke"): + calibrate( + modflow_dataset.config_file, + modflow_dataset.observed, + modflow_dataset.tmp_run_dir("one-step"), + modflow_dataset.settings(), + ) + + assert not [record for record in caplog.records if "dis.nstp" in record.getMessage()] + + +class TestModflowNamesWithoutModflow: + @pytest.mark.unit + def test_a_modflow_name_is_refused_when_the_configuration_does_not_enable_it( + self, dataset, no_pool + ): + with pytest.raises(ValueError, match="does not enable MODFLOW"): + dataset.calibrate(bounds={SPECIFIC_YIELD: (0.05, 0.3)}) + + assert not dataset.run_dir.exists() + assert no_pool == [] + + @pytest.mark.unit + def test_a_run_without_modflow_leaves_the_path_alone(self, dataset, broken_search, monkeypatch): + calls = [] + monkeypatch.setattr(_deps, "ensure_mf2005_on_path", lambda: calls.append("path")) + + with pytest.raises(RuntimeError, match="the search broke"): + dataset.calibrate() + + assert calls == [] + + @pytest.mark.unit + def test_the_message_of_a_dead_worker_is_the_one_of_today(self, dataset, monkeypatch): + monkeypatch.setattr( + scipy_optimize, + "differential_evolution", + _record_then_raise(BrokenProcessPool("A process in the process pool was terminated")), + ) + + with pytest.raises(CalibrationError) as failure: + dataset.calibrate() + + assert "dis.nstp" not in str(failure.value) + + +class TestRecordOrder: + @pytest.mark.unit + def test_the_modflow_values_order_candidates_that_share_the_nine_parameters(self): + nine = dict.fromkeys(CALIBRATION_PARAMETERS, 0.5) + records = [ + {"id": "a", "parameters": {**nine, SPECIFIC_YIELD: 0.3}}, + {"id": "b", "parameters": {**nine, SPECIFIC_YIELD: 0.1}}, + ] + + ordered = sorted(records, key=runner._record_order) + + assert [record["id"] for record in ordered] == ["b", "a"] diff --git a/tests/unit/calibration/test_worker.py b/tests/unit/calibration/test_worker.py index 35948e6..b4823b7 100644 --- a/tests/unit/calibration/test_worker.py +++ b/tests/unit/calibration/test_worker.py @@ -12,12 +12,15 @@ from rubem.calibration._worker import ( EvaluationContext, _derived_document, + _document_for, evaluate, simulate_best, ) +from rubem.calibration.modflow_parameters import catalog from rubem.calibration.objective import INADMISSIBLE_OBJECTIVE, read_series from rubem.calibration.parameters import ( FREE_PARAMETERS, + decision_space, parameters_to_vector, vector_to_parameters, ) @@ -26,6 +29,7 @@ VARIABLE_IDS, ModelConfigurationFileV1, ) +from rubem.configuration.modflow_configuration import ModflowSettings from tests.helpers.synthetic import write_synthetic_dataset TIMESTEPS = 3 @@ -381,3 +385,164 @@ def test_a_variable_the_run_does_not_write_raises(self, dataset, tmp_path): str(output_dir), "nope", ) + + +SPECIFIC_YIELD = "modflow.layers.1.specific_yield" +KH = "modflow.layers.1.kh.2" + + +class ModflowDocument: + """A calibrated document with a MODFLOW section, and the catalog of that section. + + The rasters of the section are never read: the model is replaced by a spy + that keeps the document it is given. + """ + + def __init__(self, dataset, tmp_path): + table = tmp_path / "kh1.tbl" + table.write_text("1 0.5\n2 0.1\n", encoding="utf8") + layer = { + "name": "upper", + "bottom": "/maps/bottom.map", + "initial_head": "/maps/head.map", + "boundary": "/maps/bound.map", + "laytype": 1, + "horizontal_conductivity": {"map": "/maps/classes.map", "table": str(table)}, + "vertical_conductivity": 0.1, + "specific_yield": 0.15, + } + river = { + "layers": [1], + "stage": "/maps/stage.map", + "bottom": "/maps/riv_bottom.map", + "conductance": 0.387, + "mask": "/maps/riv_mask.map", + } + settings = ModflowSettings.model_validate( + { + "enabled": True, + "top": "/maps/top.map", + "layers": [layer], + "river": {"enabled": True, "entries": [river]}, + } + ) + self.table = table + self.catalog = catalog(settings) + self.document = {**dataset.document, "modflow": settings.model_dump(mode="json")} + self.parameters = { + **vector_to_parameters(dataset.vector), + SPECIFIC_YIELD: 0.2, + KH: 0.25, + } + + +@pytest.fixture +def modflow_document(dataset, tmp_path): + return ModflowDocument(dataset, tmp_path) + + +@pytest.fixture +def model_spy(monkeypatch): + """Keep the document the model is built from and stop before the run.""" + documents = [] + + def from_config(document, **kwargs): + documents.append(json.loads(json.dumps(document))) + output_dir = Path(document["model_simulation_output"]["dir_path"]) + documents[-1]["listing"] = sorted(entry.name for entry in output_dir.iterdir()) + raise RuntimeError("stopped before the run") + + monkeypatch.setattr(Model, "from_config", from_config) + return documents + + +class TestModflowCandidate: + @pytest.mark.unit + def test_the_modflow_values_of_the_candidate_reach_its_document( + self, modflow_document, tmp_path + ): + output_dir = tmp_path / "evaluation" + output_dir.mkdir() + + document = _document_for( + modflow_document.document, + "arn", + modflow_document.parameters, + str(output_dir), + modflow=modflow_document.catalog, + ) + + layer = document["modflow"]["layers"][0] + assert layer["specific_yield"] == 0.2 + assert Path(layer["horizontal_conductivity"]["table"]) == output_dir / "kh_layer1.tbl" + assert (output_dir / "kh_layer1.tbl").read_text("utf8").split() == ["1", "0.5", "2", "0.25"] + # The nine parameters of the model are the only ones of its section. + assert sorted(document["model_calibration_parameters"]) == sorted( + ["alpha", "b", "w_1", "w_2", "w_3", "rcd", "f", "alpha_gw", "x"] + ) + ModelConfigurationFileV1.model_validate(document) + # The calibrated document is copied, never patched. + assert modflow_document.document["modflow"]["layers"][0]["specific_yield"] == 0.15 + + @pytest.mark.unit + def test_without_a_catalog_the_section_is_carried_unchanged(self, modflow_document, tmp_path): + document = _document_for( + modflow_document.document, "arn", modflow_document.parameters, str(tmp_path) + ) + + assert document["modflow"] == modflow_document.document["modflow"] + + @pytest.mark.unit + def test_an_evaluation_runs_and_records_its_modflow_values( + self, dataset, modflow_document, model_spy + ): + space = decision_space( + bounds={SPECIFIC_YIELD: (0.05, 0.3)}, fixed={KH: 0.25}, modflow=modflow_document.catalog + ) + context = EvaluationContext( + document=modflow_document.document, + base_dir=dataset.configuration.base_dir, + variable="arn", + observed=read_series(dataset.observed_path), + spinup_steps=0, + temp_dir=str(dataset.temp_dir), + evaluations_dir=str(dataset.evaluations_dir), + space=space, + modflow=modflow_document.catalog, + ) + + value = evaluate([*dataset.vector.tolist(), 0.2], context) + + assert value == INADMISSIBLE_OBJECTIVE + (record,) = dataset.records() + assert record["error"] == "RuntimeError: stopped before the run" + assert record["parameters"][SPECIFIC_YIELD] == 0.2 + assert record["parameters"][KH] == 0.25 + (document,) = model_spy + assert document["modflow"]["layers"][0]["specific_yield"] == 0.2 + # The rewritten table lives in the evaluation directory, which is + # removed with everything the run wrote. + assert document["listing"] == ["kh_layer1.tbl"] + assert list(dataset.temp_dir.iterdir()) == [] + + @pytest.mark.unit + def test_the_best_candidate_is_run_with_its_modflow_values( + self, dataset, modflow_document, model_spy, tmp_path + ): + output_dir = tmp_path / "best" + output_dir.mkdir() + + with pytest.raises(RuntimeError, match="stopped before the run"): + simulate_best( + modflow_document.document, + dataset.configuration.base_dir, + modflow_document.parameters, + str(output_dir), + "arn", + modflow=modflow_document.catalog, + ) + + (document,) = model_spy + layer = document["modflow"]["layers"][0] + assert layer["specific_yield"] == 0.2 + assert Path(layer["horizontal_conductivity"]["table"]) == output_dir / "kh_layer1.tbl" diff --git a/tests/unit/configuration/test_modflow_documentation.py b/tests/unit/configuration/test_modflow_documentation.py new file mode 100644 index 0000000..134040e --- /dev/null +++ b/tests/unit/configuration/test_modflow_documentation.py @@ -0,0 +1,51 @@ +"""The configuration examples of the groundwater page are sections the loader accepts.""" + +import json +import re +import textwrap +from pathlib import Path + +from rubem.configuration.modflow_configuration import ModflowSettings + +PAGE = Path(__file__).resolve().parents[3] / "doc" / "source" / "groundwater.rst" + +_JSON_BLOCK = re.compile(r"^\.\. code-block:: json\n\n((?:(?: .*)?\n)+)", re.MULTILINE) + + +def json_examples() -> list[dict]: + """Every ``code-block:: json`` of the page, parsed.""" + text = PAGE.read_text(encoding="utf-8") + return [json.loads(textwrap.dedent(block)) for block in _JSON_BLOCK.findall(text)] + + +def sections() -> list[dict]: + """The ``modflow`` (format 1.0) or ``MODFLOW`` (legacy) sections of the examples.""" + found = [] + for example in json_examples(): + for key in ("modflow", "MODFLOW"): + if key in example: + found.append(example[key]) + return found + + +def test_the_page_has_json_examples(): + assert json_examples() + + +def test_the_page_shows_an_enabled_section(): + assert any(section.get("enabled") is True for section in sections()) + + +def test_every_section_of_the_page_is_valid(): + found = sections() + assert found + for section in found: + ModflowSettings.model_validate(section) + + +def test_the_layer_names_of_the_template_are_top_down(): + """The example lists the layers from the top down: layer 1 is the upper one.""" + template = next(section for section in sections() if section.get("enabled") is True) + settings = ModflowSettings.model_validate(template) + assert [layer.name for layer in settings.layers] == ["upper", "middle", "lower"] + assert settings.pcraster_layer(1) == 3 diff --git a/tests/unit/configuration/test_modflow_settings.py b/tests/unit/configuration/test_modflow_settings.py new file mode 100644 index 0000000..5073a09 --- /dev/null +++ b/tests/unit/configuration/test_modflow_settings.py @@ -0,0 +1,951 @@ +"""The optional MODFLOW section: schema, layer numbering and wiring in both formats.""" + +import copy +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from rubem.cli import main +from rubem.configuration.migrate import migrate_legacy_file +from rubem.configuration.model_configuration import ModelConfiguration +from rubem.configuration.model_configuration_file import ModelConfigurationFile +from rubem.configuration.model_configuration_file_v1 import ModelConfigurationFileV1 +from rubem.configuration.modflow_configuration import ModflowSettings +from tests.helpers.synthetic import write_synthetic_dataset + + +def layer(name, bottom, head, laytype=2, **overrides): + """One layer of the section, with every file named after the layer.""" + return { + "name": name, + "bottom": bottom, + "initial_head": head, + "boundary": "modflow/bound.map", + "laytype": laytype, + "horizontal_conductivity": { + "map": f"modflow/kh_classes_{name}.map", + "table": f"modflow/kh_{name}.tbl", + }, + "vertical_conductivity": f"modflow/kv_{name}.map", + "specific_yield": 0.15, + "specific_storage": 1e-6, + **overrides, + } + + +def section(**overrides): + """The target configuration of the plan: three layers listed top down.""" + data = { + "enabled": True, + "top": "modflow/top_model.map", + "layers": [ + layer("upper", "modflow/botton3.map", "modflow/head3.map", laytype=1), + layer("middle", "modflow/botton2.map", "modflow/head2.map"), + layer("lower", "modflow/botton.map", "modflow/head1.map"), + ], + "dis": {"nstp": 5, "tsmult": 1.0, "steady_state": False}, + "solver": { + "mxiter": 2000, + "iter1": 20, + "npcond": 1, + "hclose": 5.0, + "rclose": 3.0, + "relax": 1.0, + "nbpol": 2, + "damp": 0.5, + }, + "wetting": { + "enabled": True, + "map": "modflow/wet.map", + "layers": [1], + "wetfct": 1.0, + "iwetit": 3, + "ihdwet": 0, + }, + "river": { + "enabled": True, + "entries": [ + { + "layers": [1], + "stage": "modflow/riv_stage.map", + "bottom": "modflow/riv_bot.map", + "conductance": 0.387, + "mask": "modflow/riv_cond.map", + } + ], + }, + "ghb": { + "enabled": True, + "entries": [ + { + "layers": [1, 2, 3], + "head": "modflow/ghb_head.map", + "conductance": "modflow/ghb_cond.map", + } + ], + }, + "drain": {"enabled": False, "entries": []}, + "coupling": { + "dynamic_root_depth": { + "enabled": False, + "minimum_depth_table": None, + "water_table": {"method": "highest_active_head", "layer": None}, + } + }, + "output": { + "heads": True, + "river_leakage": True, + "storage": False, + "drain_flow": False, + "root_depth": False, + }, + } + data.update(overrides) + return data + + +def refused(data): + """The validation message of ``data``.""" + with pytest.raises(ValidationError) as error: + ModflowSettings.model_validate(data) + return str(error.value) + + +class TestSchema: + @pytest.mark.unit + def test_the_target_configuration_validates(self): + settings = ModflowSettings.model_validate(section()) + + assert settings.enabled is True + assert [item.name for item in settings.layers] == ["upper", "middle", "lower"] + assert settings.layers[0].horizontal_conductivity.table == "modflow/kh_upper.tbl" + assert settings.river.entries[0].conductance == 0.387 + assert settings.ghb.entries[0].layers == [1, 2, 3] + assert settings.dis.steady_state is False + + @pytest.mark.unit + def test_the_defaults_are_the_scientist_values_with_one_time_step(self): + # One time step per stress period: a solver failure then raises instead + # of ending the process (the extension reads the heads of the period + # only when its last time step converged). + settings = ModflowSettings() + + assert settings.enabled is False + assert settings.top is None and settings.layers == [] + assert (settings.dis.nstp, settings.dis.tsmult, settings.dis.steady_state) == ( + 1, + 1.0, + False, + ) + solver = settings.solver + assert (solver.mxiter, solver.iter1, solver.npcond, solver.nbpol) == (2000, 20, 1, 2) + assert (solver.hclose, solver.rclose, solver.relax, solver.damp) == (5.0, 3.0, 1.0, 0.5) + assert settings.wetting.enabled is False + assert (settings.wetting.wetfct, settings.wetting.iwetit, settings.wetting.ihdwet) == ( + 1.0, + 3, + 0, + ) + assert not (settings.river.enabled or settings.ghb.enabled or settings.drain.enabled) + assert settings.coupling.dynamic_root_depth.enabled is False + output = settings.output + assert (output.heads, output.river_leakage) == (True, True) + assert not (output.storage or output.drain_flow or output.root_depth) + + @pytest.mark.unit + @pytest.mark.parametrize("flag, expected", [(1, True), (0, False)]) + def test_enabled_is_a_boolean_that_accepts_one_and_zero(self, flag, expected): + data = section(enabled=flag) + data["wetting"]["enabled"] = flag + data["river"]["enabled"] = 1 + + settings = ModflowSettings.model_validate(data) + + assert settings.enabled is expected + assert settings.wetting.enabled is expected + assert settings.river.enabled is True + + @pytest.mark.unit + def test_the_settings_are_frozen(self): + settings = ModflowSettings.model_validate(section()) + + with pytest.raises(ValidationError): + settings.enabled = False + + @pytest.mark.unit + @pytest.mark.parametrize( + "path, key", + [ + ((), "wells"), + ((), "recharge"), + ((), "bottom"), + (("dis",), "time_unit"), + (("dis",), "length_unit"), + (("solver",), "type"), + (("solver",), "fail_on_non_convergence"), + (("coupling",), "baseflow_from_river_leakage"), + (("wetting",), "source_boundary_layer"), + (("wetting",), "multiplier"), + (("layers", 0), "top"), + (("river",), "layers"), + ], + ) + def test_the_keys_the_prototype_had_are_refused_by_name(self, path, key): + data = section() + target = data + for step in path: + target = target[step] + target[key] = 1 + + assert key in refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize( + "laytype", [0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33] + ) + def test_every_bcf_layer_type_is_accepted(self, laytype): + data = section(enabled=False) + data["layers"][0]["laytype"] = laytype + + assert ModflowSettings.model_validate(data).layers[0].laytype == laytype + + @pytest.mark.unit + @pytest.mark.parametrize("laytype", [-1, 4, 9, 14, 40, 34]) + def test_other_layer_types_are_refused(self, laytype): + data = section(enabled=False) + data["layers"][0]["laytype"] = laytype + + assert "LAYTYPE" in refused(data) + + @pytest.mark.unit + def test_a_layer_type_must_be_given(self): + data = section() + del data["layers"][1]["laytype"] + + assert "laytype" in refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize( + "value, expected", + [ + ("modflow/kh.map", "modflow/kh.map"), + (0.5, 0.5), + (2, 2.0), + ({"map": "classes.map", "table": "kh.tbl"}, None), + ], + ) + def test_the_horizontal_conductivity_is_a_map_a_number_or_a_lookup(self, value, expected): + data = section() + data["layers"][1]["horizontal_conductivity"] = value + + conductivity = ModflowSettings.model_validate(data).layers[1].horizontal_conductivity + + if expected is None: + assert (conductivity.map, conductivity.table) == ("classes.map", "kh.tbl") + else: + assert conductivity == expected + + @pytest.mark.unit + @pytest.mark.parametrize( + "field", ["horizontal_conductivity", "vertical_conductivity", "specific_storage"] + ) + @pytest.mark.parametrize("value", [-1.0, float("nan"), float("inf")]) + def test_a_numeric_property_must_be_finite_and_physical(self, field, value): + data = section() + data["layers"][1][field] = value + + refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize("field", ["horizontal_conductivity", "vertical_conductivity"]) + def test_a_numeric_conductivity_must_be_positive(self, field): + data = section() + data["layers"][1][field] = 0.0 + + refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize("value", [-0.1, 1.5]) + def test_a_numeric_specific_yield_is_a_fraction(self, value): + data = section() + data["layers"][1]["specific_yield"] = value + + refused(data) + + @pytest.mark.unit + def test_a_numeric_string_stays_a_path(self): + data = section() + data["layers"][1]["vertical_conductivity"] = "0.5" + + assert ModflowSettings.model_validate(data).layers[1].vertical_conductivity == "0.5" + + @pytest.mark.unit + def test_an_empty_path_is_refused(self): + data = section() + data["layers"][1]["bottom"] = "" + + refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize("value", [0, -1.0]) + def test_a_numeric_river_conductance_must_be_positive(self, value): + data = section() + data["river"]["entries"][0]["conductance"] = value + + refused(data) + + +class TestEnabledRules: + """Cross-field rules, checked only when the section is enabled.""" + + @pytest.mark.unit + def test_a_disabled_section_is_not_checked(self): + data = section(enabled=False, top=None) + data["river"]["enabled"] = False + data["layers"][1]["laytype"] = 1 + data["wetting"]["layers"] = [9] + + assert ModflowSettings.model_validate(data).enabled is False + + @pytest.mark.unit + def test_an_empty_disabled_section_is_accepted(self): + assert ModflowSettings.model_validate({"enabled": False}).layers == [] + + @pytest.mark.unit + def test_the_model_top_is_required(self): + assert "'top'" in refused(section(top=None)) + + @pytest.mark.unit + def test_at_least_one_layer_is_required(self): + assert "at least one layer" in refused(section(layers=[])) + + @pytest.mark.unit + def test_layer_names_are_unique(self): + data = section() + data["layers"][2]["name"] = "upper" + + assert "'upper'" in refused(data) + + @pytest.mark.unit + def test_at_most_nine_layers_are_accepted(self): + # mfh1 at step 10 and mfh10 at step 1 would share one output name. + data = section() + data["layers"] += [ + layer(f"deep{number}", f"modflow/bottom{number}.map", "modflow/head1.map") + for number in range(4, 11) + ] + + message = refused(data) + + assert "at most 9 layers" in message + assert "10" in message + + @pytest.mark.unit + def test_nine_layers_are_accepted(self): + data = section() + data["layers"] += [ + layer(f"deep{number}", f"modflow/bottom{number}.map", "modflow/head1.map") + for number in range(4, 10) + ] + + assert len(ModflowSettings.model_validate(data).layers) == 9 + + @pytest.mark.unit + def test_the_river_package_is_required(self): + data = section() + data["river"]["enabled"] = False + + assert "river" in refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize("package", ["river", "ghb", "drain"]) + def test_an_enabled_package_needs_an_entry(self, package): + data = section() + data[package] = {"enabled": True, "entries": []} + + assert f"{package}.enabled" in refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize( + "package, entry", + [ + ("river", {"stage": "s.map", "bottom": "b.map", "conductance": "c.map"}), + ("ghb", {"head": "h.map", "conductance": "c.map"}), + ("drain", {"elevation": "e.map", "conductance": "c.map"}), + ], + ) + @pytest.mark.parametrize("number", [4, 7]) + def test_a_package_layer_must_exist(self, package, entry, number): + data = section() + data[package] = {"enabled": True, "entries": [{**entry, "layers": [number]}]} + + message = refused(data) + + assert f"layer {number}" in message and "1-3" in message + + @pytest.mark.unit + @pytest.mark.parametrize("number", [0, -1]) + def test_a_layer_number_is_positive(self, number): + data = section() + data["ghb"]["entries"][0]["layers"] = [number] + + refused(data) + + @pytest.mark.unit + def test_an_entry_lists_at_least_one_layer(self): + data = section() + data["ghb"]["entries"][0]["layers"] = [] + + refused(data) + + @pytest.mark.unit + def test_a_layer_appears_once_in_an_entry(self): + data = section() + data["ghb"]["entries"][0]["layers"] = [1, 2, 2] + + assert "ghb" in refused(data) + + @pytest.mark.unit + def test_a_layer_appears_in_one_entry_per_package(self): + data = section() + first = data["river"]["entries"][0] + data["river"]["entries"].append({**first, "layers": [2, 1]}) + + message = refused(data) + + assert "river" in message and "layer 1" in message + + @pytest.mark.unit + def test_the_same_layer_may_carry_several_packages(self): + data = section() + data["drain"] = { + "enabled": True, + "entries": [{"layers": [1], "elevation": "e.map", "conductance": "c.map"}], + } + + assert ModflowSettings.model_validate(data).drain.entries[0].layers == [1] + + @pytest.mark.unit + @pytest.mark.parametrize( + "laycon, missing", + [ + (0, "specific_storage"), + (1, "specific_yield"), + (2, "specific_storage"), + (2, "specific_yield"), + (3, "specific_storage"), + (3, "specific_yield"), + ], + ) + def test_a_transient_run_needs_the_storage_of_the_layer_type(self, laycon, missing): + data = section() + data["wetting"]["enabled"] = False + target = data["layers"][0] + target["laytype"] = 10 + laycon + target[missing] = None + + message = refused(data) + + assert missing in message and f"LAYCON {laycon}" in message and "layer 1" in message + + @pytest.mark.unit + @pytest.mark.parametrize("laycon, unused", [(0, "specific_yield"), (1, "specific_storage")]) + def test_a_transient_run_does_not_need_the_other_storage(self, laycon, unused): + data = section() + data["wetting"]["enabled"] = False + data["layers"][0]["laytype"] = laycon + data["layers"][0][unused] = None + + assert ModflowSettings.model_validate(data).layers[0].laytype == laycon + + @pytest.mark.unit + def test_a_steady_state_run_needs_no_storage(self): + data = section(dis={"steady_state": True}) + for item in data["layers"]: + item["specific_yield"] = None + item["specific_storage"] = None + + assert ModflowSettings.model_validate(data).dis.steady_state is True + + @pytest.mark.unit + def test_storage_output_needs_a_transient_run(self): + # The extension ends the process when asked for storage in a steady-state run. + data = section(dis={"steady_state": True}, output={"storage": True}) + + message = refused(data) + + assert "output.storage" in message and "steady_state" in message + + @pytest.mark.unit + def test_storage_output_is_accepted_in_a_transient_run(self): + data = section(output={"storage": True}) + + assert ModflowSettings.model_validate(data).output.storage is True + + @pytest.mark.unit + @pytest.mark.parametrize("laytype", [1, 11, 21, 31]) + def test_laycon_1_is_valid_only_on_the_top_layer(self, laytype): + data = section() + data["layers"][2]["laytype"] = laytype + + message = refused(data) + + assert "layer 3" in message and "LAYCON 1" in message + + @pytest.mark.unit + def test_wetting_on_a_layer_that_is_not_laycon_1_or_3_is_refused(self): + data = section() + data["wetting"]["layers"] = [1, 2, 3] + + message = refused(data) + + assert "layer 2" in message and "LAYCON 2" in message + assert "layer 3" in message + + @pytest.mark.unit + def test_wetting_on_every_layer_needs_laycon_1_or_3(self): + data = section() + data["layers"][1]["laytype"] = 3 + data["layers"][2]["laytype"] = 33 + data["wetting"]["layers"] = [1, 2, 3] + + assert ModflowSettings.model_validate(data).wetting_layers() == [1, 2, 3] + + @pytest.mark.unit + def test_wetting_layers_must_exist(self): + data = section() + data["wetting"]["layers"] = [1, 5] + + assert "layer 5" in refused(data) + + @pytest.mark.unit + def test_a_wetting_layer_is_listed_once(self): + data = section() + data["wetting"]["layers"] = [1, 1] + + assert "wetting" in refused(data) + + @pytest.mark.unit + def test_wetting_without_layers_applies_to_every_laycon_1_or_3_layer(self): + data = section() + data["layers"][2]["laytype"] = 13 + data["wetting"]["layers"] = None + + assert ModflowSettings.model_validate(data).wetting_layers() == [1, 3] + + @pytest.mark.unit + def test_wetting_without_layers_needs_a_laycon_1_or_3_layer(self): + data = section() + data["layers"][0]["laytype"] = 2 + data["wetting"]["layers"] = None + + assert "LAYCON 1 or 3" in refused(data) + + @pytest.mark.unit + def test_wetting_needs_its_map(self): + data = section() + data["wetting"]["map"] = None + + assert "wetting.map" in refused(data) + + @pytest.mark.unit + def test_no_wetting_layer_when_wetting_is_disabled(self): + data = section() + data["wetting"]["enabled"] = False + + assert ModflowSettings.model_validate(data).wetting_layers() == [] + + @pytest.mark.unit + def test_a_numeric_river_conductance_needs_a_mask(self): + data = section() + data["river"]["entries"][0]["mask"] = None + + assert "mask" in refused(data) + + @pytest.mark.unit + def test_a_river_conductance_map_needs_no_mask(self): + data = section() + entry = data["river"]["entries"][0] + entry["conductance"] = "modflow/riv_cond.map" + entry["mask"] = None + + settings = ModflowSettings.model_validate(data) + + assert settings.river.entries[0].conductance == "modflow/riv_cond.map" + + @pytest.mark.unit + def test_the_root_depth_coupling_needs_its_minimum_depth_table(self): + data = section() + data["coupling"]["dynamic_root_depth"]["enabled"] = True + + assert "minimum_depth_table" in refused(data) + + @pytest.mark.unit + @pytest.mark.parametrize("number, valid", [(None, False), (2, True), (4, False)]) + def test_the_water_table_layer_method_needs_a_layer_in_range(self, number, valid): + data = section() + data["coupling"]["dynamic_root_depth"] = { + "enabled": True, + "minimum_depth_table": "txt/Dpz_min.txt", + "water_table": {"method": "layer", "layer": number}, + } + + if valid: + settings = ModflowSettings.model_validate(data) + assert settings.coupling.dynamic_root_depth.water_table.layer == number + else: + assert "water_table" in refused(data) + + @pytest.mark.unit + def test_a_water_table_layer_is_only_given_with_the_layer_method(self): + data = section() + data["coupling"]["dynamic_root_depth"] = { + "enabled": True, + "minimum_depth_table": "txt/Dpz_min.txt", + "water_table": {"method": "highest_active_head", "layer": 2}, + } + + assert "water_table.layer" in refused(data) + + @pytest.mark.unit + def test_the_highest_unconfined_method_needs_a_convertible_layer(self): + data = section(dis={"steady_state": True}) + data["wetting"]["enabled"] = False + for item in data["layers"]: + item["laytype"] = 0 + data["coupling"]["dynamic_root_depth"] = { + "enabled": True, + "minimum_depth_table": "txt/Dpz_min.txt", + "water_table": {"method": "highest_unconfined"}, + } + + assert "highest_unconfined" in refused(data) + + @pytest.mark.unit + def test_every_violation_is_reported_at_once(self): + data = section(top=None) + data["layers"][2]["name"] = "upper" + data["river"]["enabled"] = False + + message = refused(data) + + assert "'top'" in message and "'upper'" in message and "river" in message + + +class TestLayerNumbering: + """User layers are numbered top down (MODFLOW), PCRaster numbers them bottom up.""" + + @pytest.mark.unit + @pytest.mark.parametrize("user, pcraster", [(1, 3), (2, 2), (3, 1)]) + def test_the_top_layer_is_the_last_pcraster_layer(self, user, pcraster): + settings = ModflowSettings.model_validate(section()) + + assert settings.pcraster_layer(user) == pcraster + assert settings.user_layer(pcraster) == user + + @pytest.mark.unit + @pytest.mark.parametrize("number", [0, 4]) + def test_a_number_outside_the_layers_is_refused(self, number): + settings = ModflowSettings.model_validate(section()) + + with pytest.raises(ValueError, match="1-3"): + settings.pcraster_layer(number) + with pytest.raises(ValueError, match="1-3"): + settings.user_layer(number) + + +class TestPaths: + @pytest.mark.unit + def test_every_relative_path_is_anchored_and_numbers_are_kept(self, tmp_path): + data = section() + data["layers"][1]["specific_yield"] = "modflow/sy.map" + data["layers"][2]["horizontal_conductivity"] = 0.3 + data["drain"] = { + "enabled": True, + "entries": [{"layers": [2], "elevation": "drn/e.map", "conductance": "drn/c.map"}], + } + data["coupling"]["dynamic_root_depth"]["minimum_depth_table"] = "txt/Dpz_min.txt" + + anchored = ModflowSettings.model_validate(data).resolve_paths(tmp_path) + + def at(relative): + return tmp_path / relative + + assert Path(anchored.top) == at("modflow/top_model.map") + upper, middle, lower = anchored.layers + assert Path(upper.bottom) == at("modflow/botton3.map") + assert Path(upper.initial_head) == at("modflow/head3.map") + assert Path(upper.boundary) == at("modflow/bound.map") + assert Path(upper.horizontal_conductivity.map) == at("modflow/kh_classes_upper.map") + assert Path(upper.horizontal_conductivity.table) == at("modflow/kh_upper.tbl") + assert Path(upper.vertical_conductivity) == at("modflow/kv_upper.map") + assert (upper.specific_yield, upper.specific_storage) == (0.15, 1e-6) + assert Path(middle.specific_yield) == at("modflow/sy.map") + assert lower.horizontal_conductivity == 0.3 + assert Path(anchored.wetting.map) == at("modflow/wet.map") + river = anchored.river.entries[0] + assert Path(river.stage) == at("modflow/riv_stage.map") + assert Path(river.bottom) == at("modflow/riv_bot.map") + assert Path(river.mask) == at("modflow/riv_cond.map") + assert river.conductance == 0.387 + ghb = anchored.ghb.entries[0] + assert (Path(ghb.head), Path(ghb.conductance)) == ( + at("modflow/ghb_head.map"), + at("modflow/ghb_cond.map"), + ) + drain = anchored.drain.entries[0] + assert (Path(drain.elevation), Path(drain.conductance)) == ( + at("drn/e.map"), + at("drn/c.map"), + ) + table = anchored.coupling.dynamic_root_depth.minimum_depth_table + assert Path(table) == at("txt/Dpz_min.txt") + + @pytest.mark.unit + def test_absolute_paths_and_absent_files_are_kept(self, tmp_path): + data = section() + absolute = str(tmp_path / "elsewhere" / "top.map") + data["top"] = absolute + data["river"]["entries"][0]["conductance"] = "modflow/riv_cond.map" + data["river"]["entries"][0]["mask"] = None + + anchored = ModflowSettings.model_validate(data).resolve_paths(tmp_path / "base") + + assert anchored.top == absolute + assert anchored.river.entries[0].mask is None + assert anchored.wetting.layers == [1] + assert anchored.coupling.dynamic_root_depth.minimum_depth_table is None + + @pytest.mark.unit + def test_no_base_directory_keeps_the_settings(self): + settings = ModflowSettings.model_validate(section()) + + assert settings.resolve_paths(None) is settings + + @pytest.mark.unit + def test_the_names_and_methods_are_not_paths(self, tmp_path): + anchored = ModflowSettings.model_validate(section()).resolve_paths(tmp_path) + + assert [item.name for item in anchored.layers] == ["upper", "middle", "lower"] + assert anchored.coupling.dynamic_root_depth.water_table.method == "highest_active_head" + + @pytest.mark.unit + def test_a_disabled_section_is_anchored_too(self, tmp_path): + data = section(enabled=False) + + anchored = ModflowSettings.model_validate(data).resolve_paths(tmp_path) + + assert Path(anchored.top) == tmp_path / "modflow" / "top_model.map" + assert anchored == ModflowSettings.model_validate( + copy.deepcopy(anchored.model_dump(mode="json")) + ) + + +@pytest.fixture(name="config") +def config_fixture(tmp_path): + """A legacy configuration of the synthetic dataset, with relative MODFLOW paths possible.""" + return write_synthetic_dataset(str(tmp_path)) + + +def both_formats(config): + """The legacy configuration and its format 1.0 document.""" + legacy = ModelConfigurationFile.model_validate(config) + return (config, ModelConfigurationFileV1.from_legacy(legacy).to_dict()) + + +class TestLegacyFile: + @pytest.mark.unit + def test_the_section_is_absent_by_default_and_written_as_null(self, config): + file = ModelConfigurationFile.model_validate(config) + + assert file.modflow is None + assert file.to_dict()["MODFLOW"] is None + assert ModelConfigurationFile.model_validate(file.to_dict()) == file + + @pytest.mark.unit + def test_the_section_round_trips(self, config): + config["MODFLOW"] = section() + + file = ModelConfigurationFile.model_validate(config) + + assert file.modflow == ModflowSettings.model_validate(section()) + assert file.to_dict()["MODFLOW"]["layers"][0]["name"] == "upper" + assert ModelConfigurationFile.model_validate(file.to_dict()) == file + + @pytest.mark.unit + def test_the_lower_case_spelling_is_accepted(self, config): + config["modflow"] = section() + + assert ModelConfigurationFile.model_validate(config).modflow.enabled is True + + @pytest.mark.unit + @pytest.mark.parametrize("flag, expected", [(1, True), (0, False)]) + def test_enabled_accepts_one_and_zero(self, config, flag, expected): + config["MODFLOW"] = section(enabled=flag) + + assert ModelConfigurationFile.model_validate(config).modflow.enabled is expected + + @pytest.mark.unit + def test_an_unknown_key_of_the_section_is_refused(self, config): + config["MODFLOW"] = section(wells={"enabled": 0}) + + with pytest.raises(ValidationError, match="wells"): + ModelConfigurationFile.model_validate(config) + + @pytest.mark.unit + def test_the_section_paths_are_anchored(self, config, tmp_path): + config["MODFLOW"] = section() + + anchored = ModelConfigurationFile.model_validate(config).resolve_paths(tmp_path) + + assert Path(anchored.modflow.top) == tmp_path / "modflow" / "top_model.map" + lookup = anchored.modflow.layers[2].horizontal_conductivity + assert Path(lookup.table) == tmp_path / "modflow" / "kh_lower.tbl" + + @pytest.mark.unit + def test_anchoring_without_the_section_keeps_it_absent(self, config, tmp_path): + anchored = ModelConfigurationFile.model_validate(config).resolve_paths(tmp_path) + + assert anchored.modflow is None + + +class TestFormat10File: + @pytest.mark.unit + def test_the_section_is_left_out_when_absent(self, config): + document = both_formats(config)[1] + + assert "modflow" not in document + assert ModelConfigurationFileV1.model_validate(document).modflow is None + + @pytest.mark.unit + def test_the_section_follows_the_conversion_from_and_to_legacy(self, config): + config["MODFLOW"] = section() + legacy = ModelConfigurationFile.model_validate(config) + + v1 = ModelConfigurationFileV1.from_legacy(legacy) + + assert v1.modflow == legacy.modflow + assert v1.to_legacy() == legacy + + @pytest.mark.unit + def test_the_document_round_trips_with_unset_keys(self, config): + config["MODFLOW"] = section() + document = both_formats(config)[1] + + model = ModelConfigurationFileV1.model_validate(document) + + assert "minimum_depth_table" not in document["modflow"]["coupling"]["dynamic_root_depth"] + assert model.modflow.coupling.dynamic_root_depth.minimum_depth_table is None + assert ModelConfigurationFileV1.model_validate(model.to_dict()) == model + + @pytest.mark.unit + @pytest.mark.parametrize("flag, expected", [(1, True), (0, False)]) + def test_enabled_accepts_one_and_zero(self, config, flag, expected): + document = both_formats(config)[1] + document["modflow"] = section(enabled=flag) + + assert ModelConfigurationFileV1.model_validate(document).modflow.enabled is expected + + @pytest.mark.unit + def test_an_unknown_key_of_the_section_is_refused(self, config): + document = both_formats(config)[1] + document["modflow"] = section(recharge={"option": 3}) + + with pytest.raises(ValidationError, match="recharge"): + ModelConfigurationFileV1.model_validate(document) + + @pytest.mark.unit + def test_the_section_paths_are_anchored(self, config, tmp_path): + document = both_formats(config)[1] + document["modflow"] = section() + + anchored = ModelConfigurationFileV1.model_validate(document).resolve_paths(tmp_path) + + assert Path(anchored.modflow.river.entries[0].stage) == ( + tmp_path / "modflow" / "riv_stage.map" + ) + + +class TestModelConfiguration: + @pytest.mark.unit + def test_an_absent_section_disables_the_coupling_silently(self, config): + for source in both_formats(config): + loaded = ModelConfiguration(source, validate_input=False) + + assert loaded.modflow is None + assert loaded.modflow_enabled is False + assert not [p for p in loaded.problems if "MODFLOW" in p.description] + + @pytest.mark.unit + def test_an_enabled_section_reaches_the_configuration_anchored(self, config, tmp_path): + """The section names files that do not exist: only their absence is reported.""" + config["MODFLOW"] = section() + + for source in both_formats(config): + loaded = ModelConfiguration( + source, validate_input=False, base_dir=tmp_path, allow_blocking_problems=True + ) + + assert loaded.modflow_enabled is True + assert Path(loaded.modflow.top) == tmp_path / "modflow" / "top_model.map" + modflow = [p for p in loaded.problems if "MODFLOW" in p.description] + assert {p.description for p in modflow} == {"MODFLOW input file does not exist."} + assert Path(modflow[0].file) == tmp_path / "modflow" / "top_model.map" + + @pytest.mark.unit + @pytest.mark.parametrize("validate_input", [True, False]) + def test_a_disabled_section_is_reported_as_ignored(self, config, validate_input): + config["MODFLOW"] = section(enabled=False) + + for source in both_formats(config): + loaded = ModelConfiguration(source, validate_input=validate_input) + + assert loaded.modflow_enabled is False + assert loaded.modflow.enabled is False + ignored = [p for p in loaded.problems if "MODFLOW section is ignored" in p.description] + assert len(ignored) == 1 and not ignored[0].blocking + assert "enabled" in ignored[0].reason + + +class TestCommands: + @pytest.mark.unit + @pytest.mark.parametrize( + "arguments, key", + [ + (["config", "schema"], "modflow"), + (["config", "schema", "--format", "legacy"], "MODFLOW"), + ], + ) + def test_the_schema_describes_the_section(self, capsys, restore_logging, arguments, key): + main(arguments) + + schema = json.loads(capsys.readouterr().out) + assert key in schema["properties"] + assert "ModflowSettings" in schema["$defs"] + assert "ModflowLayer" in schema["$defs"] + assert not [name for name in schema["$defs"] if "__" in name] + + @pytest.mark.unit + def test_migration_carries_the_section_with_paths_relative_to_the_destination( + self, config, tmp_path + ): + data = section() + data["layers"][1]["specific_yield"] = "modflow/sy.map" + config["MODFLOW"] = data + source = tmp_path / "legacy.json" + source.write_text(json.dumps(config), encoding="utf8") + + written = migrate_legacy_file(source, tmp_path / "migrated" / "config-v1.json") + + document = json.loads(Path(written).read_text(encoding="utf8")) + modflow = document["modflow"] + assert Path(modflow["top"]) == Path("..") / "modflow" / "top_model.map" + lookup = modflow["layers"][0]["horizontal_conductivity"] + assert Path(lookup["table"]) == Path("..") / "modflow" / "kh_upper.tbl" + assert Path(modflow["layers"][1]["specific_yield"]) == Path("..") / "modflow" / "sy.map" + assert modflow["layers"][0]["specific_yield"] == 0.15 + assert modflow["river"]["entries"][0]["conductance"] == 0.387 + migrated = ModelConfigurationFileV1.from_json(written).resolve_paths(Path(written).parent) + assert Path(migrated.modflow.top).resolve() == (tmp_path / "modflow" / "top_model.map") diff --git a/tests/unit/core/test_dynamic_model_modflow.py b/tests/unit/core/test_dynamic_model_modflow.py new file mode 100644 index 0000000..a55c033 --- /dev/null +++ b/tests/unit/core/test_dynamic_model_modflow.py @@ -0,0 +1,451 @@ +"""The dynamic model coupled to a mocked PCRaster MODFLOW extension. + +The synthetic dataset runs two monthly steps (January 2000, 31 days, and +February 2000, 29 days) with the three-layer section of +:func:`write_modflow_inputs`. The extension is a :class:`unittest.mock.Mock` +whose getters return fields the test controls, so the coupling formulas are +checked against known heads and leakages without ``mf2005``; the runs against +the real executable live in ``tests/integration/test_modflow_coupling.py``. +""" + +import subprocess +import sys +from pathlib import Path +from unittest.mock import Mock + +import numpy as np +import pcraster as pcr +import pytest + +from rubem.configuration.model_configuration import ModelConfiguration +from rubem.core import DynamicFrameworkWrapper +from rubem.hydrological_processes import Evapotranspiration, _modflow +from tests.helpers.compare import compare_csv, compare_rasters +from tests.helpers.synthetic import ( + CELL_SIZE, + ROOTZONE_DEPTH, + geotiff_series_name, + series_name, + write_minimum_root_depth_table, + write_modflow_inputs, + write_synthetic_dataset, +) +from tests.unit.core.test_core import expected_outputs, run_model + +pytestmark = pytest.mark.unit + +CELL_AREA = CELL_SIZE * CELL_SIZE +DAYS = {1: 31, 2: 29} +LEAKAGE = -2.0 # m3/day, out of the aquifer on every cell +MINIMUM_ROOT_DEPTH = 30.0 # cm + + +def read(path): + """A PCRaster map or a GeoTIFF as an array (missing: NaN).""" + return pcr.pcr2numpy(pcr.scalar(pcr.readmap(str(path))), np.nan) + + +def dem(config): + pcr.setclone(config["RASTERS"]["clone"]) + return read(config["RASTERS"]["dem"]) + + +def modflow_outputs(directory): + return sorted(path.name for path in Path(directory).iterdir() if path.name.startswith("mf")) + + +class Extension: + """The mocked extension and what the model did with it.""" + + def __init__(self, monkeypatch): + self.mf = Mock() + self.mf.converged.return_value = True + self.mf.getRiverLeakage.side_effect = lambda layer: pcr.spatial(pcr.scalar(LEAKAGE)) + self.mf.getStorage.side_effect = lambda layer: pcr.spatial(pcr.scalar(10.0 * layer)) + self.mf.getDrain.side_effect = lambda layer: pcr.spatial(pcr.scalar(-1.0)) + self.depth_below_dem = 10.0 # m, the head of every layer below the DEM + self.dem = None + self.mf.getHeads.side_effect = self._heads + self.run_directories = [] + self.mf.run.side_effect = self._run + monkeypatch.setattr(_modflow, "_initialise", lambda clone: self.mf) + monkeypatch.setattr(_modflow, "ensure_mf2005_on_path", lambda: Path("mf2005")) + monkeypatch.setattr("rubem.validation.modflow_inputs.missing_groundwater_deps", lambda: []) + + def _heads(self, layer): + return pcr.numpy2pcr(pcr.Scalar, self.dem - self.depth_below_dem, np.nan) + + def _run(self, directory): + path = Path(directory) + assert path.is_dir(), "MODFLOW runs in a directory that exists" + self.run_directories.append(path) + + def recharge(self): + """The recharge rate given to the extension at each step [m/day].""" + return [pcr.pcr2numpy(call.args[0], np.nan) for call in self.mf.setRecharge.call_args_list] + + +@pytest.fixture(name="extension") +def extension_fixture(monkeypatch): + return Extension(monkeypatch) + + +def coupled_config(directory, extension, **section_updates): + config = write_synthetic_dataset(str(directory)) + section = write_modflow_inputs(config) + section.update(section_updates) + config["MODFLOW"] = section + extension.dem = dem(config) + return config + + +def root_depth_coupling(config): + return { + "enabled": True, + "minimum_depth_table": write_minimum_root_depth_table(config, MINIMUM_ROOT_DEPTH), + "water_table": {"method": "highest_active_head"}, + } + + +def run(config): + wrapper = DynamicFrameworkWrapper.load(ModelConfiguration(config, validate_input=False)) + wrapper.run() + return wrapper.dynamic_model_concept + + +class TestBaseflow: + def test_the_baseflow_is_the_aquifer_to_river_leakage_of_the_step(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + + run(config) + + out = tmp_path / "out" + for step, days in DAYS.items(): + expected = -LEAKAGE * days * 1000.0 / CELL_AREA + np.testing.assert_allclose(read(out / series_name("bfw", step)), expected, rtol=1e-6) + + def test_the_recharge_of_each_step_reaches_modflow_per_day(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + + run(config) + + out = tmp_path / "out" + rates = extension.recharge() + assert len(rates) == len(DAYS) + for (step, days), rate in zip(DAYS.items(), rates, strict=True): + recharge = read(out / series_name("rec", step)) + np.testing.assert_allclose(rate, recharge / (1000.0 * days), rtol=1e-6) + periods = [call.args[0] for call in extension.mf.updateDISParameter.call_args_list] + assert periods == list(DAYS.values()) + + def test_the_saturated_zone_is_frozen(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + + model = run(config) + + initial = config["INITIAL_SOIL_CONDITIONS"] + storage = pcr.pcr2numpy(pcr.spatial(model.current_soil_sat_zone_storage), np.nan) + np.testing.assert_array_equal(storage, np.float32(initial["s_sat_ini"])) + baseflow = pcr.pcr2numpy(pcr.spatial(model.previous_baseflow), np.nan) + np.testing.assert_array_equal(baseflow, np.float32(initial["bfw_ini"])) + + +class TestRunDirectory: + def test_modflow_runs_under_the_output_directory_and_it_is_removed(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + + model = run(config) + + run_directory = tmp_path / "out" / "modflow" + assert extension.run_directories == [run_directory] * len(DAYS) + assert not run_directory.exists() + assert model.modflow is None + + def test_a_failed_step_keeps_the_run_directory(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + extension.mf.converged.return_value = False + + with pytest.raises(RuntimeError, match="did not converge in stress period 1"): + run(config) + + assert (tmp_path / "out" / "modflow").is_dir() + + def test_an_existing_run_directory_is_refused_and_kept(self, tmp_path, extension): + # The run removes its directory at the end, so it may not adopt one. + config = coupled_config(tmp_path, extension) + notes = tmp_path / "out" / "modflow" / "notes.txt" + notes.parent.mkdir(parents=True) + notes.write_text("kept", encoding="utf8") + + with pytest.raises(RuntimeError, match="already exists") as error: + run(config) + + assert str(notes.parent) in str(error.value) + assert notes.read_text(encoding="utf8") == "kept" + assert extension.mf.mock_calls == [] + + def test_a_failed_start_removes_the_new_run_directory(self, tmp_path, extension, monkeypatch): + config = coupled_config(tmp_path, extension) + + def missing_extension(clone): + raise RuntimeError("no extension") + + monkeypatch.setattr(_modflow, "_initialise", missing_extension) + + with pytest.raises(RuntimeError, match="no extension"): + run(config) + + assert (tmp_path / "out").is_dir() + assert not (tmp_path / "out" / "modflow").exists() + + def test_the_default_run_creates_no_run_directory(self, tmp_path): + run_model(str(tmp_path)) + + assert not (tmp_path / "out" / "modflow").exists() + + +class TestDiagnostics: + def test_heads_and_river_exchange_are_written_in_both_formats(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + + run(config) + + out = tmp_path / "out" + for step in DAYS: + for prefix in ("mfh1", "mfh2", "mfh3", "mfaq2rv", "mfrv2aq", "mfrvnet"): + assert (out / series_name(prefix, step)).is_file(), (prefix, step) + assert (out / geotiff_series_name(prefix, step)).is_file(), (prefix, step) + np.testing.assert_allclose(read(out / series_name("mfaq2rv", step)), -LEAKAGE) + np.testing.assert_allclose(read(out / series_name("mfrv2aq", step)), 0.0) + np.testing.assert_allclose(read(out / series_name("mfrvnet", step)), LEAKAGE) + np.testing.assert_allclose( + read(out / series_name("mfh1", step)), extension.dem - 10.0, rtol=1e-6 + ) + assert not [name for name in modflow_outputs(out) if name.startswith(("mfst", "mfdrn"))] + + def test_storage_and_drain_flow_are_written_per_layer(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + section = config["MODFLOW"] + section["drain"] = { + "enabled": True, + "entries": [ + { + "layers": [2], + "elevation": section["layers"][1]["bottom"], + "conductance": section["ghb"]["entries"][0]["conductance"], + } + ], + } + section["output"] = {"storage": True, "drain_flow": True} + + run(config) + + out = tmp_path / "out" + for step in DAYS: + for number in (1, 2, 3): + storage = read(out / series_name(f"mfst{number}", step)) + # User layer n is extension layer 4 - n. + np.testing.assert_allclose(storage, 10.0 * (4 - number)) + np.testing.assert_allclose(read(out / series_name("mfdrn2", step)), -1.0) + assert not [name for name in modflow_outputs(out) if name.startswith(("mfdrn1", "mfdrn3"))] + + def test_disabled_diagnostics_are_not_written(self, tmp_path, extension): + config = coupled_config( + tmp_path, extension, output={"heads": False, "river_leakage": False} + ) + + run(config) + + assert modflow_outputs(tmp_path / "out") == [] + + def test_nothing_is_written_without_a_raster_format(self, tmp_path, extension): + config = coupled_config(tmp_path, extension, output={"storage": True}) + config["MODFLOW"]["coupling"] = {"dynamic_root_depth": root_depth_coupling(config)} + config["MODFLOW"]["output"]["root_depth"] = True + config["RASTER_FILE_FORMAT"] = {"map_raster_series": False, "tiff_raster_series": False} + config["GENERATE_FILE"] = {key: False for key in config["GENERATE_FILE"]} + + run(config) + + assert sorted(path.name for path in (tmp_path / "out").iterdir()) == [] + + +class TestRootDepthCoupling: + """The previous step's water table restricts the vegetation root depth.""" + + @pytest.fixture(name="stress") + def stress_fixture(self, monkeypatch): + """The soil water stress coefficient of the vegetated and the bare-soil area, per call.""" + calls = {"vegetated": [], "bare": [], "arguments": []} + vegetated_coef = Evapotranspiration.get_water_stress_coef_et_vegetated_area + vegetated_et = Evapotranspiration.get_et_vegetated_area + bare_et = Evapotranspiration.get_water_stress_coef_et_bare_soil_area + + def coef(moisture, wilting_point, field_capacity): + calls["arguments"].append( + [ + pcr.pcr2numpy(pcr.spatial(pcr.scalar(item)), np.nan) + for item in (moisture, wilting_point, field_capacity) + ] + ) + return vegetated_coef(moisture, wilting_point, field_capacity) + + def vegetated(potential, crop, water_stress): + calls["vegetated"].append(pcr.pcr2numpy(pcr.spatial(water_stress), np.nan)) + return vegetated_et(potential, crop, water_stress) + + def bare(potential, crop, water_stress): + calls["bare"].append(pcr.pcr2numpy(pcr.spatial(water_stress), np.nan)) + return bare_et(potential, crop, water_stress) + + monkeypatch.setattr( + Evapotranspiration, "get_water_stress_coef_et_vegetated_area", staticmethod(coef) + ) + monkeypatch.setattr(Evapotranspiration, "get_et_vegetated_area", staticmethod(vegetated)) + monkeypatch.setattr( + Evapotranspiration, "get_water_stress_coef_et_bare_soil_area", staticmethod(bare) + ) + return calls + + def coupled_run(self, directory, extension, depth_below_dem, coupling=True): + config = coupled_config(directory, extension, output={"root_depth": True}) + if coupling: + config["MODFLOW"]["coupling"] = {"dynamic_root_depth": root_depth_coupling(config)} + extension.depth_below_dem = depth_below_dem + run(config) + return directory / "out" + + def test_the_root_depth_follows_the_previous_water_table(self, tmp_path, extension): + """Head 0.5 m below the DEM: 50 cm of roots at step 2, none of it at step 1.""" + out = self.coupled_run(tmp_path, extension, depth_below_dem=0.5) + + np.testing.assert_allclose(read(out / series_name("mfwt", 1)), extension.dem - 0.5) + np.testing.assert_allclose(read(out / series_name("mfzr", 1)), ROOTZONE_DEPTH, rtol=1e-6) + np.testing.assert_allclose(read(out / series_name("mfzfrac", 1)), 1.0) + assert np.isnan(read(out / series_name("mfgwd", 1))).all() + np.testing.assert_allclose(read(out / series_name("mfgwd", 2)), 50.0, rtol=1e-4) + np.testing.assert_allclose(read(out / series_name("mfzr", 2)), 50.0, rtol=1e-4) + np.testing.assert_allclose( + read(out / series_name("mfzfrac", 2)), 50.0 / ROOTZONE_DEPTH, rtol=1e-4 + ) + for step in DAYS: + for prefix in ("mfwt", "mfzr", "mfzfrac", "mfgwd"): + assert (out / geotiff_series_name(prefix, step)).is_file(), (prefix, step) + + def test_the_minimum_root_depth_bounds_a_shallow_water_table(self, tmp_path, extension): + out = self.coupled_run(tmp_path, extension, depth_below_dem=0.1) + + np.testing.assert_allclose(read(out / series_name("mfgwd", 2)), 10.0, rtol=1e-3) + np.testing.assert_allclose( + read(out / series_name("mfzr", 2)), MINIMUM_ROOT_DEPTH, rtol=1e-6 + ) + + def test_a_head_above_the_ground_counts_as_no_depth(self, tmp_path, extension): + out = self.coupled_run(tmp_path, extension, depth_below_dem=-2.0) + + np.testing.assert_allclose(read(out / series_name("mfgwd", 2)), 0.0) + np.testing.assert_allclose( + read(out / series_name("mfzr", 2)), MINIMUM_ROOT_DEPTH, rtol=1e-6 + ) + + def test_a_cell_without_a_valid_head_keeps_every_root(self, tmp_path, extension): + extension.mf.getHeads.side_effect = lambda layer: pcr.spatial(pcr.scalar(_modflow.DRY_HEAD)) + + config = coupled_config(tmp_path, extension, output={"root_depth": True}) + config["MODFLOW"]["coupling"] = {"dynamic_root_depth": root_depth_coupling(config)} + run(config) + + out = tmp_path / "out" + np.testing.assert_allclose(read(out / series_name("mfzr", 2)), ROOTZONE_DEPTH, rtol=1e-6) + np.testing.assert_allclose(read(out / series_name("mfzfrac", 2)), 1.0) + + def test_only_the_vegetation_feels_the_restricted_roots(self, tmp_path, extension, stress): + """Step 2: the vegetated area sees the root fraction, the bare soil does not.""" + self.coupled_run(tmp_path / "plain", extension, depth_below_dem=0.5, coupling=False) + plain = {key: list(values) for key, values in stress.items()} + for values in stress.values(): + values.clear() + self.coupled_run(tmp_path / "coupled", extension, depth_below_dem=0.5) + coupled = stress + + # The coupled run computes two coefficients per step: soil, then vegetation. + assert len(plain["arguments"]) == len(DAYS) + assert len(coupled["arguments"]) == 2 * len(DAYS) + # Step 1 has no previous head: both runs are the same. + np.testing.assert_array_equal(coupled["vegetated"][0], plain["vegetated"][0]) + np.testing.assert_array_equal(coupled["bare"][0], plain["bare"][0]) + # Step 2: the soil coefficient is unchanged and feeds the bare soil ... + np.testing.assert_array_equal(coupled["bare"][1], plain["bare"][1]) + np.testing.assert_array_equal(coupled["arguments"][2], plain["arguments"][1]) + # ... and the vegetation takes the coefficient of the storages scaled by the fraction. + fraction = np.float32(50.0 / ROOTZONE_DEPTH) + scaled = coupled["arguments"][3] + for full, restricted in zip(plain["arguments"][1], scaled, strict=True): + np.testing.assert_allclose(restricted, full * fraction, rtol=1e-4) + assert not np.allclose(coupled["vegetated"][1], plain["vegetated"][1]) + + def test_a_deep_water_table_changes_nothing(self, tmp_path, extension): + """Heads 10 m below the DEM leave the roots (Zr about 1.5 m) whole.""" + plain = self.coupled_run( + tmp_path / "plain", extension, depth_below_dem=10.0, coupling=False + ) + coupled = self.coupled_run(tmp_path / "coupled", extension, depth_below_dem=10.0) + + for name in expected_outputs(): + compare = compare_csv if name.endswith(".csv") else compare_rasters + result = compare(plain / name, coupled / name, rtol=0.0, atol=0.0) + assert result.equal, f"{name}:\n{result.report()}" + + def test_the_root_depth_diagnostics_need_the_output_flag(self, tmp_path, extension): + config = coupled_config(tmp_path, extension) + config["MODFLOW"]["coupling"] = {"dynamic_root_depth": root_depth_coupling(config)} + + run(config) + + written = modflow_outputs(tmp_path / "out") + assert not [name for name in written if name.startswith(("mfwt", "mfzr", "mfgwd"))] + + def test_the_step_fields_are_released(self, tmp_path, extension): + config = coupled_config(tmp_path, extension, output={"root_depth": True}) + config["MODFLOW"]["coupling"] = {"dynamic_root_depth": root_depth_coupling(config)} + + model = run(config) + + released = ("effective_root_depth", "root_depth_fraction", "groundwater_depth_cm") + assert [name for name in released if getattr(model, name) is not None] == [] + for name in ("soil_rootzone_depth_min", "surface_elevation", "previous_water_table_head"): + assert getattr(model, name) is not None, name + + +class TestDefaultPath: + def test_an_absent_or_disabled_section_gives_the_same_outputs(self, tmp_path): + """No output moves when the section is absent, disabled or empty and disabled.""" + reference = tmp_path / "absent" + run_model(str(reference)) + + disabled = tmp_path / "disabled" + config = write_synthetic_dataset(str(disabled)) + config["MODFLOW"] = {**write_modflow_inputs(config), "enabled": False} + run_model(str(disabled), config=config) + + empty = tmp_path / "empty" + config = write_synthetic_dataset(str(empty)) + config["MODFLOW"] = {"enabled": False} + run_model(str(empty), config=config) + + for other in (disabled, empty): + names = sorted(path.name for path in (other / "out").iterdir()) + assert names == sorted(path.name for path in (reference / "out").iterdir()) + for name in expected_outputs(): + compare = compare_csv if name.endswith(".csv") else compare_rasters + result = compare(reference / "out" / name, other / "out" / name, rtol=0.0, atol=0.0) + assert result.equal, f"{other.name} {name}:\n{result.report()}" + + def test_the_dynamic_model_does_not_import_the_groundwater_module(self): + code = ( + "import sys, rubem._dynamic_model; " + "sys.exit('rubem.hydrological_processes._modflow' in sys.modules)" + ) + completed = subprocess.run([sys.executable, "-c", code], check=False) + + assert completed.returncode == 0 diff --git a/tests/unit/hydrological_processes/test_module_modflow.py b/tests/unit/hydrological_processes/test_module_modflow.py new file mode 100644 index 0000000..c6f2652 --- /dev/null +++ b/tests/unit/hydrological_processes/test_module_modflow.py @@ -0,0 +1,732 @@ +"""The MODFLOW groundwater module against a mocked PCRaster MODFLOW extension. + +The inputs are real PCRaster maps on the 3x3 synthetic grid (pyfakefs cannot +back PCRaster I/O); the extension object is a :class:`unittest.mock.Mock`, so +the tests see every call the module makes and the numbers it passes. With the +three layers of :func:`write_modflow_inputs`, user layer 1 (the top) is +PCRaster layer 3 and user layer 3 (the base) is PCRaster layer 1. +""" + +import math +from pathlib import Path +from unittest.mock import Mock + +import numpy as np +import pcraster as pcr +import pytest + +from rubem.configuration.modflow_configuration import ModflowSettings +from rubem.hydrological_processes import _modflow +from rubem.hydrological_processes._modflow import DRY_HEAD, ModflowGroundwater +from tests.helpers.synthetic import ( + CELL_SIZE, + COLS, + MODFLOW_HEAD, + MODFLOW_KH_TABLE, + ROWS, + write_grid_map, + write_modflow_inputs, + write_synthetic_dataset, +) + +pytestmark = pytest.mark.unit + +NAN = math.nan +CELL_AREA = CELL_SIZE * CELL_SIZE +DAYS = 31 +LEFT_COLUMN = [1.0 if index % COLS == 0 else 0.0 for index in range(ROWS * COLS)] +MIDDLE_COLUMN = [1.0 if index % COLS == 1 else 0.0 for index in range(ROWS * COLS)] + + +@pytest.fixture(name="section") +def section_fixture(tmp_path): + """The MODFLOW section of :func:`write_modflow_inputs` (three layers).""" + config = write_synthetic_dataset(str(tmp_path)) + return write_modflow_inputs(config) + + +@pytest.fixture(name="mf") +def mf_fixture(): + mf = Mock() + mf.converged.return_value = True + mf.getRiverLeakage.side_effect = lambda layer: pcr.spatial(pcr.scalar(-2.0)) + mf.getHeads.side_effect = lambda layer: pcr.spatial(pcr.scalar(float(layer))) + mf.getStorage.side_effect = lambda layer: pcr.spatial(pcr.scalar(10.0 * layer)) + mf.getDrain.side_effect = lambda layer: pcr.spatial(pcr.scalar(-100.0)) + mf.getGeneralHeadLeakage.side_effect = lambda layer: pcr.spatial(pcr.scalar(-100.0)) + return mf + + +@pytest.fixture(name="extension") +def extension_fixture(monkeypatch, mf): + """Replace the extension and the PATH lookup; records the order they are used in.""" + events = [] + + def initialise(clone): + events.append("initialise") + return mf + + def ensure_mf2005_on_path(): + events.append("ensure_mf2005_on_path") + return Path("mf2005") + + monkeypatch.setattr(_modflow, "_initialise", initialise) + monkeypatch.setattr(_modflow, "ensure_mf2005_on_path", ensure_mf2005_on_path) + return events + + +@pytest.fixture(name="run_dir") +def run_dir_fixture(tmp_path): + return tmp_path / "run" / "modflow" + + +def build(section, run_dir, days=DAYS, area=CELL_AREA): + model = ModflowGroundwater(ModflowSettings.model_validate(section), area, run_dir) + model.initialize(days) + return model + + +def values(field): + return pcr.pcr2numpy(pcr.scalar(field), np.nan) + + +def grid(values_): + return np.asarray(values_, dtype=float).reshape(ROWS, COLS) + + +def calls(mf, name): + return [item for item in mf.mock_calls if item[0] == name] + + +def layer_args(mf, name, pcraster_number, position=-1): + """The arguments of the last ``name`` call for extension layer ``pcraster_number``.""" + matching = [item.args for item in calls(mf, name) if item.args[position] == pcraster_number] + assert matching, f"{name} was not called for layer {pcraster_number}" + return matching[-1] + + +def rewrite(path, values_, nominal=False): + write_grid_map(path, values_, nominal=nominal) + + +def give_layers_own_boundaries(section, active_in): + """Cell (1, 1) active only in the layers ``active_in``, every other cell everywhere.""" + for number, layer in enumerate(section["layers"], start=1): + layer["boundary"] = write_grid_map( + Path(section["top"]).with_name(f"bound{number}.map"), + [1 if index or number in active_in else 0 for index in range(9)], + nominal=True, + ) + + +class TestConstruction: + @pytest.mark.parametrize("area", [0.0, -1.0, math.nan, math.inf]) + def test_the_cell_area_must_be_finite_and_positive(self, section, run_dir, area): + settings = ModflowSettings.model_validate(section) + with pytest.raises(ValueError, match="cell area"): + ModflowGroundwater(settings, area, run_dir) + + def test_a_disabled_section_is_refused(self, run_dir): + with pytest.raises(ValueError, match="not enabled"): + ModflowGroundwater(ModflowSettings(), CELL_AREA, run_dir) + + +@pytest.mark.usefixtures("extension") +class TestInitialize: + def test_the_packages_are_set_dis_bas_bcf_wetting_solver_stress(self, section, run_dir, mf): + groups = { + "createBottomLayer": 0, + "addLayer": 0, + "setRowWidth": 0, + "setColumnWidth": 0, + "setDISParameter": 0, + "setBoundary": 1, + "setInitialHead": 1, + "setDryHead": 2, + "setConductivity": 2, + "setStorage": 2, + "setWettingParameter": 3, + "setWetting": 3, + "setPCG": 4, + "setRiver": 5, + "setGeneralHead": 5, + "setDrain": 5, + } + build(section, run_dir) + ranks = [groups[item[0]] for item in mf.mock_calls] + assert ranks == sorted(ranks) + assert set(ranks) == set(range(6)) + + def test_the_surfaces_reach_the_extension_from_the_bottom_up(self, section, run_dir, mf): + build(section, run_dir) + base, above_base = mf.createBottomLayer.call_args.args + np.testing.assert_array_equal(values(base), np.full((ROWS, COLS), 40.0)) + np.testing.assert_array_equal(values(above_base), np.full((ROWS, COLS), 60.0)) + tops = [values(item.args[0]) for item in calls(mf, "addLayer")] + np.testing.assert_array_equal( + tops, [np.full((ROWS, COLS), 80.0), np.full((ROWS, COLS), 100.0)] + ) + + def test_columns_inactive_in_every_layer_get_synthetic_elevations(self, section, run_dir, mf): + inactive_left = [ + NAN if index == 0 else 0.0 if index % COLS == 0 else 1.0 for index in range(9) + ] + rewrite(section["layers"][0]["boundary"], inactive_left, nominal=True) + for number, layer in enumerate(section["layers"], start=1): + rewrite( + layer["bottom"], + [NAN if index % COLS == 0 else 100.0 - 20.0 * number for index in range(9)], + ) + rewrite(section["top"], [NAN if index % COLS == 0 else 100.0 for index in range(9)]) + build(section, run_dir) + base, above_base = mf.createBottomLayer.call_args.args + surfaces = [base, above_base] + [item.args[0] for item in calls(mf, "addLayer")] + for index, surface in enumerate(surfaces): + np.testing.assert_array_equal(values(surface)[:, 0], [float(index)] * ROWS) + np.testing.assert_array_equal( + values(surface)[:, 1:], np.full((ROWS, COLS - 1), 40.0 + 20.0 * index) + ) + boundary = pcr.pcr2numpy(layer_args(mf, "setBoundary", 3)[0], -99) + np.testing.assert_array_equal(boundary[:, 0], [0, 0, 0]) + + def test_a_column_active_in_an_upper_layer_only_needs_every_elevation( + self, section, run_dir, mf + ): + give_layers_own_boundaries(section, active_in={1}) + rewrite(section["layers"][2]["bottom"], [NAN if index == 0 else 40.0 for index in range(9)]) + with pytest.raises(ValueError, match=r"bottom of layer 3 \(layer3\).*row 1, column 1"): + build(section, run_dir) + mf.createBottomLayer.assert_not_called() + + def test_a_column_active_in_the_base_layer_only_keeps_its_elevations( + self, section, run_dir, mf + ): + give_layers_own_boundaries(section, active_in={3}) + build(section, run_dir) + base, above_base = mf.createBottomLayer.call_args.args + surfaces = [base, above_base] + [item.args[0] for item in calls(mf, "addLayer")] + assert [float(values(surface)[0, 0]) for surface in surfaces] == [40.0, 60.0, 80.0, 100.0] + + @pytest.mark.parametrize("surface", ["top", "bottom"]) + def test_a_missing_elevation_in_an_active_column_is_refused( + self, section, run_dir, mf, surface + ): + path = section["top"] if surface == "top" else section["layers"][2]["bottom"] + level = 100.0 if surface == "top" else 40.0 + rewrite(path, [NAN if index == 1 else level for index in range(9)]) + with pytest.raises(ValueError, match="row 1, column 2"): + build(section, run_dir) + mf.createBottomLayer.assert_not_called() + + def test_a_bottom_above_the_surface_over_it_is_refused(self, section, run_dir, mf): + rewrite( + section["layers"][1]["bottom"], [90.0 if index == 4 else 60.0 for index in range(9)] + ) + with pytest.raises(ValueError, match="row 2, column 2"): + build(section, run_dir) + mf.createBottomLayer.assert_not_called() + + def test_user_layers_reach_the_extension_bottom_up(self, section, run_dir, mf): + """The extension numbers the layers bottom up, and takes them in that order. + + It appends the LAYCON of each ``setConductivity`` call to the BCF file in + call order, whatever layer number the call names: called top down, the + top layer would get the LAYCON of the base. + """ + build(section, run_dir) + assert [item.args[-1] for item in calls(mf, "setBoundary")] == [1, 2, 3] + assert [item.args[-1] for item in calls(mf, "setInitialHead")] == [1, 2, 3] + assert [ + (item.args[0], item.args[3], item.args[4]) for item in calls(mf, "setConductivity") + ] == [ + (2, 1, True), + (2, 2, True), + (1, 3, True), + ] + assert [item.args[-1] for item in calls(mf, "setStorage")] == [1, 2, 3] + assert [item.args[-1] for item in calls(mf, "setWetting")] == [3] + assert [item.args[-1] for item in calls(mf, "setRiver")] == [3] + assert sorted(item.args[-1] for item in calls(mf, "setGeneralHead")) == [1, 2, 3] + + def test_the_initial_head_is_read_from_its_map(self, section, run_dir, mf): + build(section, run_dir) + head = layer_args(mf, "setInitialHead", 3)[0] + np.testing.assert_array_equal(values(head), np.full((ROWS, COLS), MODFLOW_HEAD)) + + def test_the_conductivity_classes_are_looked_up_in_the_table(self, section, run_dir, mf): + build(section, run_dir) + kh = layer_args(mf, "setConductivity", 3, position=3)[1] + expected = [MODFLOW_KH_TABLE[1]] * COLS + [MODFLOW_KH_TABLE[2]] * (ROWS - 1) * COLS + np.testing.assert_allclose(values(kh), grid(expected)) + + def test_a_rewritten_table_is_read_again(self, section, run_dir, mf): + build(section, run_dir) + table = Path(section["layers"][0]["horizontal_conductivity"]["table"]) + table.write_text("1 2.0\n2 3.0\n", encoding="utf8") + build(section, run_dir.parent / "second") + kh = layer_args(mf, "setConductivity", 3, position=3)[1] + np.testing.assert_allclose(values(kh), grid([2.0] * COLS + [3.0] * (ROWS - 1) * COLS)) + + def test_a_number_is_spread_over_the_clone(self, section, run_dir, mf): + build(section, run_dir) + kv = layer_args(mf, "setConductivity", 2, position=3)[2] + np.testing.assert_allclose(values(kv), np.full((ROWS, COLS), 0.1)) + + def test_a_missing_property_on_an_active_cell_is_refused(self, section, run_dir, mf): + rewrite( + section["layers"][0]["initial_head"], + [NAN if index == 5 else 90.0 for index in range(9)], + ) + with pytest.raises(ValueError, match="row 2, column 3"): + build(section, run_dir) + + def test_a_property_missing_only_on_inactive_cells_is_filled(self, section, run_dir, mf): + rewrite( + section["layers"][0]["boundary"], + [0 if index == 5 else 1 for index in range(9)], + nominal=True, + ) + rewrite( + section["layers"][0]["initial_head"], + [NAN if index == 5 else 90.0 for index in range(9)], + ) + build(section, run_dir) + head = values(layer_args(mf, "setInitialHead", 3)[0]) + assert np.isfinite(head).all() + + def test_a_transient_run_sets_the_storage_of_each_layer_type(self, section, run_dir, mf): + section["layers"][2]["laytype"] = 0 + build(section, run_dir) + stored = [ + (float(values(item.args[0])[0, 0]), float(values(item.args[1])[0, 0]), item.args[2]) + for item in calls(mf, "setStorage") + ] + assert stored == [ + (pytest.approx(1e-5), pytest.approx(1e-5), 1), + (pytest.approx(1e-5), pytest.approx(0.15), 2), + (pytest.approx(0.15), pytest.approx(0.15), 3), + ] + + def test_a_transient_run_starts_with_the_first_period(self, section, run_dir, mf): + build(section, run_dir, days=28) + mf.setDISParameter.assert_called_once_with(4, 2, 28.0, 1, 1.0, 0) + + def test_a_steady_run_sets_no_storage(self, section, run_dir, mf): + section["dis"] = {"steady_state": True, "nstp": 2, "tsmult": 1.5} + build(section, run_dir) + mf.setDISParameter.assert_called_once_with(4, 2, 31.0, 2, 1.5, 1) + mf.setStorage.assert_not_called() + + @pytest.mark.parametrize("area", [900.0, CELL_AREA]) + def test_the_cells_are_squares_of_the_grid_area_whatever_the_clone_units( + self, section, run_dir, mf, area + ): + build(section, run_dir, area=area) + width = math.sqrt(area) + mf.setRowWidth.assert_called_once_with([width] * ROWS) + mf.setColumnWidth.assert_called_once_with([width] * COLS) + + def test_the_solver_takes_the_configured_pcg_settings(self, section, run_dir, mf): + section["solver"] = {"mxiter": 50, "hclose": 0.01, "damp": 1.0} + build(section, run_dir) + mf.setPCG.assert_called_once_with(50, 20, 1, 0.01, 3.0, 1.0, 2, 1.0) + mf.setDryHead.assert_called_once_with(DRY_HEAD) + + def test_wetting_is_set_with_its_parameters_on_the_listed_layers(self, section, run_dir, mf): + section["wetting"].update({"wetfct": 0.5, "iwetit": 2, "ihdwet": 1}) + build(section, run_dir) + mf.setWettingParameter.assert_called_once_with(0.5, 2, 1) + (wetdry, layer), _ = mf.setWetting.call_args + assert layer == 3 + np.testing.assert_allclose(values(wetdry), np.full((ROWS, COLS), 1.0)) + + def test_wetting_reaches_the_extension_bottom_up(self, section, run_dir, mf): + section["layers"][1]["laytype"] = 3 + section["wetting"]["layers"] = [1, 2] + build(section, run_dir) + assert [item.args[-1] for item in calls(mf, "setWetting")] == [2, 3] + + def test_disabled_wetting_is_not_set(self, section, run_dir, mf): + section["wetting"]["enabled"] = False + build(section, run_dir) + mf.setWettingParameter.assert_not_called() + mf.setWetting.assert_not_called() + + def test_river_conductance_is_zero_outside_the_mask_and_the_boundary( + self, section, run_dir, mf + ): + rewrite( + section["layers"][0]["boundary"], + [0 if index == 7 else 1 for index in range(9)], + nominal=True, + ) + build(section, run_dir) + stage, bottom, conductance, layer = mf.setRiver.call_args.args + expected = [10.0 if index in (1, 4) else 0.0 for index in range(9)] + np.testing.assert_allclose(values(conductance), grid(expected)) + np.testing.assert_allclose(values(stage)[:2, 1], [95.0, 95.0]) + np.testing.assert_allclose(values(bottom)[:2, 1], [90.0, 90.0]) + + def test_a_stress_map_needs_values_on_the_cells_of_the_package(self, section, run_dir, mf): + rewrite( + section["river"]["entries"][0]["stage"], + [NAN if index == 1 else 95.0 for index in range(9)], + ) + with pytest.raises(ValueError, match="row 1, column 2"): + build(section, run_dir) + + def test_a_stress_map_is_filled_outside_the_cells_of_the_package(self, section, run_dir, mf): + rewrite( + section["river"]["entries"][0]["stage"], + [95.0 if index % COLS == 1 else NAN for index in range(9)], + ) + build(section, run_dir) + assert np.isfinite(values(mf.setRiver.call_args.args[0])).all() + + def test_non_positive_conductance_is_not_a_package_cell(self, section, run_dir, mf): + conductance = section["ghb"]["entries"][0]["conductance"] + rewrite( + conductance, [-1.0 if index == 3 else value for index, value in enumerate(LEFT_COLUMN)] + ) + build(section, run_dir) + ghb = values(mf.setGeneralHead.call_args_list[0].args[1]) + np.testing.assert_allclose(ghb[:, 0], [1.0, 0.0, 1.0]) + + def test_a_package_layer_without_cells_is_not_set(self, section, run_dir, mf, caplog): + empty = write_grid_map(Path(section["top"]).with_name("empty.map"), 0.0) + head = section["ghb"]["entries"][0]["head"] + section["ghb"]["entries"] = [ + { + "layers": [1, 2], + "head": head, + "conductance": section["ghb"]["entries"][0]["conductance"], + }, + {"layers": [3], "head": head, "conductance": empty}, + ] + with caplog.at_level("WARNING"): + build(section, run_dir) + assert [item.args[-1] for item in calls(mf, "setGeneralHead")] == [3, 2] + assert "ghb" in caplog.text and "layer 3 (layer3)" in caplog.text + + def test_the_extension_starts_after_mf2005_is_on_path(self, section, run_dir, extension): + build(section, run_dir) + assert extension == ["ensure_mf2005_on_path", "initialise"] + + def test_initialize_runs_once(self, section, run_dir): + model = build(section, run_dir) + with pytest.raises(RuntimeError, match="already initialized"): + model.initialize(DAYS) + + def test_the_run_directory_is_created(self, section, run_dir): + build(section, run_dir) + assert run_dir.is_dir() + + +class TestRuntime: + def test_a_missing_executable_is_reported_before_the_extension_starts( + self, section, run_dir, monkeypatch + ): + started = [] + monkeypatch.setattr(_modflow, "ensure_mf2005_on_path", lambda: None) + monkeypatch.setattr(_modflow, "_initialise", started.append) + monkeypatch.setattr(_modflow, "missing_groundwater_deps", lambda: ["mf2005"]) + with pytest.raises(RuntimeError, match="MODFLOW-2005 executable"): + build(section, run_dir) + assert started == [] + + def test_a_missing_extension_is_reported_with_the_installation_guidance( + self, section, run_dir, monkeypatch + ): + monkeypatch.setattr(_modflow, "ensure_mf2005_on_path", lambda: Path("mf2005")) + monkeypatch.delattr(pcr, "initialise", raising=False) + monkeypatch.setattr( + _modflow, "missing_groundwater_deps", lambda: ["pcraster._pcraster_modflow"] + ) + with pytest.raises(RuntimeError, match=r"not found: pcraster\._pcraster_modflow"): + build(section, run_dir) + + +@pytest.mark.usefixtures("extension") +class TestRunStep: + def test_recharge_goes_to_the_highest_active_cell_in_metres_per_day(self, section, run_dir, mf): + model = build(section, run_dir) + recharge = pcr.numpy2pcr(pcr.Scalar, grid([NAN] + [31.0] * 8), np.nan) + model.run_step(recharge, DAYS) + field, option = mf.setRecharge.call_args.args + assert option == 3 + np.testing.assert_allclose(values(field), grid([0.0] + [0.001] * 8)) + + def test_each_transient_period_takes_its_length(self, section, run_dir, mf): + model = build(section, run_dir) + model.run_step(pcr.scalar(0.0), 28) + mf.updateDISParameter.assert_called_once_with(28.0, 1, 1.0) + + def test_a_steady_run_keeps_its_period(self, section, run_dir, mf): + section["dis"] = {"steady_state": True} + model = build(section, run_dir) + model.run_step(pcr.scalar(0.0), 28) + mf.updateDISParameter.assert_not_called() + + def test_modflow_runs_in_the_run_directory(self, section, run_dir, mf): + model = build(section, run_dir) + model.run_step(pcr.scalar(0.0), DAYS) + (directory,), _ = mf.run.call_args + assert Path(directory) == run_dir + + def test_stress_packages_are_set_once_for_every_period(self, section, run_dir, mf): + model = build(section, run_dir) + for _ in range(2): + model.run_step(pcr.scalar(0.0), DAYS) + names = [item[0] for item in mf.mock_calls] + assert names.count("setRiver") == 1 + assert names.count("setGeneralHead") == 3 + assert names.count("run") == 2 + assert max( + index + for index, name in enumerate(names) + if name.startswith("set") and name != "setRecharge" + ) < names.index("run") + + def test_a_period_that_does_not_converge_raises(self, section, run_dir, mf): + mf.converged.side_effect = [True, False] + model = build(section, run_dir) + model.run_step(pcr.scalar(0.0), DAYS) + with pytest.raises(RuntimeError, match="did not converge in stress period 2"): + model.run_step(pcr.scalar(0.0), DAYS) + + def test_the_baseflow_is_the_aquifer_to_river_leakage_in_mm(self, section, run_dir, mf): + leakage = grid([-2.0, 3.0, 0.0] * 3) + mf.getRiverLeakage.side_effect = lambda layer: pcr.numpy2pcr(pcr.Scalar, leakage, np.nan) + model = build(section, run_dir) + result = model.run_step(pcr.scalar(0.0), DAYS) + expected_mm = 2.0 * DAYS * 1000.0 / CELL_AREA + np.testing.assert_allclose(values(result.baseflow_mm), grid([expected_mm, 0.0, 0.0] * 3)) + np.testing.assert_allclose( + values(result.aquifer_to_river_m3_per_day), grid([2.0, 0.0, 0.0] * 3) + ) + np.testing.assert_allclose( + values(result.river_to_aquifer_m3_per_day), grid([0.0, 3.0, 0.0] * 3) + ) + np.testing.assert_allclose(values(result.net_river_leakage_m3_per_day), leakage) + assert [item.args for item in calls(mf, "getRiverLeakage")] == [(3,)] + + def test_general_head_and_drain_flows_never_reach_the_baseflow(self, section, run_dir, mf): + section["drain"] = { + "enabled": True, + "entries": [ + { + "layers": [2], + "elevation": section["top"], + "conductance": section["ghb"]["entries"][0]["conductance"], + } + ], + } + section["output"] = {"drain_flow": True} + model = build(section, run_dir) + result = model.run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose( + values(result.baseflow_mm), np.full((ROWS, COLS), 2.0 * DAYS * 1000.0 / CELL_AREA) + ) + + def test_the_river_leakage_is_read_only_for_layers_with_river_cells(self, section, run_dir, mf): + bound2 = write_grid_map( + Path(section["top"]).with_name("bound2.map"), + [0 if value else 1 for value in MIDDLE_COLUMN], + nominal=True, + ) + section["layers"][1]["boundary"] = bound2 + section["river"]["entries"][0]["layers"] = [1, 2] + model = build(section, run_dir) + model.run_step(pcr.scalar(0.0), DAYS) + assert [item.args for item in calls(mf, "getRiverLeakage")] == [(3,)] + + def test_heads_are_keyed_by_user_layer(self, section, run_dir, mf): + model = build(section, run_dir) + result = model.run_step(pcr.scalar(0.0), DAYS) + assert sorted(result.heads) == [1, 2, 3] + assert {number: float(values(head)[0, 0]) for number, head in result.heads.items()} == { + 1: 3.0, + 2: 2.0, + 3: 1.0, + } + + def test_heads_are_not_read_unless_requested(self, section, run_dir, mf): + section["output"] = {"heads": False} + model = build(section, run_dir) + result = model.run_step(pcr.scalar(0.0), DAYS) + assert result.heads == {} + mf.getHeads.assert_not_called() + + def test_storage_is_read_when_requested(self, section, run_dir, mf): + model = build(section, run_dir) + assert model.run_step(pcr.scalar(0.0), DAYS).storage == {} + mf.getStorage.assert_not_called() + section["output"] = {"storage": True} + model = build(section, run_dir.parent / "second") + storage = model.run_step(pcr.scalar(0.0), DAYS).storage + assert {number: float(values(item)[0, 0]) for number, item in storage.items()} == { + 1: 30.0, + 2: 20.0, + 3: 10.0, + } + + def test_the_drain_flow_is_read_only_for_layers_with_drain_cells(self, section, run_dir, mf): + empty = write_grid_map(Path(section["top"]).with_name("empty.map"), 0.0) + conductance = section["ghb"]["entries"][0]["conductance"] + section["drain"] = { + "enabled": True, + "entries": [ + {"layers": [1], "elevation": section["top"], "conductance": conductance}, + {"layers": [2], "elevation": section["top"], "conductance": empty}, + ], + } + section["output"] = {"drain_flow": True} + model = build(section, run_dir) + result = model.run_step(pcr.scalar(0.0), DAYS) + assert [item.args for item in calls(mf, "getDrain")] == [(3,)] + assert sorted(result.drain_flow) == [1, 2] + np.testing.assert_allclose(values(result.drain_flow[1]), np.full((ROWS, COLS), -100.0)) + np.testing.assert_allclose(values(result.drain_flow[2]), np.zeros((ROWS, COLS))) + + def test_the_drain_flow_is_not_read_unless_requested(self, section, run_dir, mf): + section["drain"] = { + "enabled": True, + "entries": [ + { + "layers": [1], + "elevation": section["top"], + "conductance": section["ghb"]["entries"][0]["conductance"], + } + ], + } + model = build(section, run_dir) + assert model.run_step(pcr.scalar(0.0), DAYS).drain_flow == {} + mf.getDrain.assert_not_called() + + @pytest.mark.parametrize("days", [0, -1, math.nan]) + def test_a_period_must_last_a_positive_number_of_days(self, section, run_dir, days): + model = build(section, run_dir) + with pytest.raises(ValueError, match="days"): + model.run_step(pcr.scalar(0.0), days) + + def test_a_step_needs_an_initialized_model(self, section, run_dir): + model = ModflowGroundwater(ModflowSettings.model_validate(section), CELL_AREA, run_dir) + with pytest.raises(RuntimeError, match="initialize"): + model.run_step(pcr.scalar(0.0), DAYS) + + +def enable_root_depth(section, method, layer=None): + water_table = {"method": method} if layer is None else {"method": method, "layer": layer} + section["coupling"] = { + "dynamic_root_depth": { + "enabled": True, + "minimum_depth_table": section["layers"][0]["horizontal_conductivity"]["table"], + "water_table": water_table, + } + } + + +@pytest.mark.usefixtures("extension") +class TestWaterTable: + """User layer 1 is dry in cell 1 and 95 m elsewhere, layer 2 is 85 m, layer 3 is 50 m. + + Layer 1 spans 80-100 m, layer 2 60-80 m and layer 3 40-60 m, so the head of + layer 2 lies above its top (the layer is confined there). + """ + + @pytest.fixture(autouse=True) + def heads(self, mf, section): # section sets the clone the fields need + user = { + 1: pcr.numpy2pcr(pcr.Scalar, grid([DRY_HEAD] + [95.0] * 8), np.nan), + 2: pcr.spatial(pcr.scalar(85.0)), + 3: pcr.spatial(pcr.scalar(50.0)), + } + mf.getHeads.side_effect = lambda layer: user[4 - layer] + + def test_without_the_coupling_there_is_no_water_table(self, section, run_dir): + assert build(section, run_dir).run_step(pcr.scalar(0.0), DAYS).water_table_head is None + + def test_the_highest_active_head_skips_dry_and_inactive_cells(self, section, run_dir): + bound1 = write_grid_map( + Path(section["top"]).with_name("bound1.map"), + [0 if index == 2 else 1 for index in range(9)], + nominal=True, + ) + section["layers"][0]["boundary"] = bound1 + enable_root_depth(section, "highest_active_head") + result = build(section, run_dir).run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose( + values(result.water_table_head), grid([85.0, 95.0, 85.0] + [95.0] * 6) + ) + + def test_the_water_table_uses_the_inputs_read_at_initialize( + self, section, run_dir, monkeypatch + ): + section["layers"][0]["boundary"] = write_grid_map( + Path(section["top"]).with_name("bound1.map"), + [0 if index == 2 else 1 for index in range(9)], + nominal=True, + ) + enable_root_depth(section, "highest_active_head") + model = build(section, run_dir) + + def read_field(*args, **kwargs): + raise AssertionError("an input was read again after initialize") + + monkeypatch.setattr(_modflow, "read_field", read_field) + result = model.run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose( + values(result.water_table_head), grid([85.0, 95.0, 85.0] + [95.0] * 6) + ) + + def test_the_highest_unconfined_head_skips_convertible_layers_above_their_top( + self, section, run_dir + ): + enable_root_depth(section, "highest_unconfined") + result = build(section, run_dir).run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose(values(result.water_table_head), grid([50.0] + [95.0] * 8)) + + def test_the_highest_unconfined_head_keeps_a_convertible_layer_below_its_top( + self, section, run_dir, mf + ): + # 70 m lies inside layer 2 (60-80 m): above its bottom, below its top. + user = { + 1: pcr.numpy2pcr(pcr.Scalar, grid([DRY_HEAD] + [95.0] * 8), np.nan), + 2: pcr.spatial(pcr.scalar(70.0)), + 3: pcr.spatial(pcr.scalar(50.0)), + } + mf.getHeads.side_effect = lambda layer: user[4 - layer] + enable_root_depth(section, "highest_unconfined") + result = build(section, run_dir).run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose(values(result.water_table_head), grid([70.0] + [95.0] * 8)) + + def test_the_highest_unconfined_head_skips_confined_layers(self, section, run_dir): + section["layers"][0]["laytype"] = 0 + section["wetting"]["enabled"] = False + enable_root_depth(section, "highest_unconfined") + result = build(section, run_dir).run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose(values(result.water_table_head), np.full((ROWS, COLS), 50.0)) + + def test_an_explicit_layer_gives_its_head(self, section, run_dir): + enable_root_depth(section, "layer", layer=2) + result = build(section, run_dir).run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose(values(result.water_table_head), np.full((ROWS, COLS), 85.0)) + + def test_an_explicit_layer_skips_its_dry_and_inactive_cells(self, section, run_dir): + section["layers"][0]["boundary"] = write_grid_map( + Path(section["top"]).with_name("bound1.map"), + [0 if index == 2 else 1 for index in range(9)], + nominal=True, + ) + enable_root_depth(section, "layer", layer=1) + result = build(section, run_dir).run_step(pcr.scalar(0.0), DAYS) + np.testing.assert_allclose( + values(result.water_table_head), grid([NAN, 95.0, NAN] + [95.0] * 6) + ) + + def test_the_coupling_reads_heads_that_are_not_reported(self, section, run_dir, mf): + section["output"] = {"heads": False} + enable_root_depth(section, "layer", layer=3) + result = build(section, run_dir).run_step(pcr.scalar(0.0), DAYS) + assert result.heads == {} + np.testing.assert_allclose(values(result.water_table_head), np.full((ROWS, COLS), 50.0)) diff --git a/tests/unit/test_cli_calibrate.py b/tests/unit/test_cli_calibrate.py index 984dcfd..7c554da 100644 --- a/tests/unit/test_cli_calibrate.py +++ b/tests/unit/test_cli_calibrate.py @@ -538,6 +538,47 @@ def test_the_bounds_the_fixed_parameters_and_the_stations_reach_the_settings( assert settings.strategy == "rand1bin" assert settings.polish is True + @pytest.mark.unit + def test_the_modflow_names_reach_the_settings_as_they_are_written( + self, config_path, observed_path, tmp_path, fake_calibration, restore_logging + ): + main( + [ + "calibrate", + "-c", + str(config_path), + "--observed", + str(observed_path), + "-o", + str(tmp_path / "calibration"), + "--bound", + "modflow.layers.1.specific_yield=0.05:0.3", + "--bound", + "modflow.layers.1.kh.2=0.05:0.5", + "--fix", + "modflow.river.1.conductance=0.387", + ] + ) + + settings = fake_calibration.calls[0][3] + # The command line only parses; the calibration resolves the names + # against the MODFLOW section of the configuration. + assert settings.bounds == { + "modflow.layers.1.specific_yield": (0.05, 0.3), + "modflow.layers.1.kh.2": (0.05, 0.5), + } + assert settings.fixed == {"modflow.river.1.conductance": 0.387} + + @pytest.mark.unit + def test_the_help_of_the_bounds_names_the_modflow_parameters(self, capsys, restore_logging): + with pytest.raises(SystemExit) as error: + main(["calibrate", "--help"]) + + assert error.value.code == 0 + output = " ".join(capsys.readouterr().out.split()) + assert "MODFLOW" in output + assert "modflow.layers" in output + @pytest.mark.unit def test_no_polish_is_the_default_and_can_be_written_out( self, config_path, observed_path, tmp_path, fake_calibration, restore_logging diff --git a/tests/unit/test_deps.py b/tests/unit/test_deps.py index 41bcb5a..d3b0aff 100644 --- a/tests/unit/test_deps.py +++ b/tests/unit/test_deps.py @@ -69,3 +69,108 @@ def test_missing_dependencies_are_named_with_an_actionable_extra(self, monkeypat for name in _deps._CALIBRATION_DEPENDENCIES: assert name in message assert 'pip install "rubem[calibration]"' in message + + +class TestGroundwaterDeps: + @pytest.fixture(name="no_executable_on_path") + def no_executable_on_path_fixture(self, monkeypatch, tmp_path): + """``mf2005`` is not on PATH and the interpreter prefix is an empty directory.""" + monkeypatch.setattr(_deps.shutil, "which", lambda name: None) + monkeypatch.setattr(_deps.sys, "prefix", str(tmp_path)) + return tmp_path + + @pytest.mark.unit + def test_nothing_is_missing_in_the_conda_environment(self): + """The conda-forge pcraster package ships the extension and the executable.""" + assert _deps.missing_groundwater_deps() == [] + + @pytest.mark.unit + def test_the_executable_on_path_wins(self, monkeypatch, tmp_path): + executable = tmp_path / "mf2005" + monkeypatch.setattr(_deps.shutil, "which", lambda name: str(executable)) + + assert _deps.resolve_mf2005() == executable + + @pytest.mark.unit + @pytest.mark.parametrize( + "relative", [("bin", "mf2005"), ("Library", "bin", "mf2005.exe")], ids=["unix", "windows"] + ) + def test_the_executable_is_found_under_the_interpreter_prefix( + self, no_executable_on_path, relative + ): + """conda puts mf2005 in /bin, and in /Library/bin on Windows.""" + executable = no_executable_on_path.joinpath(*relative) + executable.parent.mkdir(parents=True) + executable.write_bytes(b"") + + assert _deps.resolve_mf2005() == executable + + @pytest.mark.unit + def test_a_missing_executable_is_reported(self, no_executable_on_path): + assert _deps.resolve_mf2005() is None + assert _deps.missing_groundwater_deps() == ["mf2005"] + + @pytest.mark.unit + def test_a_missing_extension_is_reported(self, monkeypatch): + monkeypatch.setattr(_deps.importlib.util, "find_spec", lambda name: None) + + assert _deps.missing_groundwater_deps() == ["pcraster._pcraster_modflow"] + + @pytest.mark.unit + def test_a_broken_pcraster_counts_as_a_missing_extension(self, monkeypatch): + """Looking up a submodule imports its parent package, which may raise.""" + + def raise_import_error(name): + raise ImportError(name) + + monkeypatch.setattr(_deps.importlib.util, "find_spec", raise_import_error) + + assert _deps.missing_groundwater_deps() == ["pcraster._pcraster_modflow"] + + @pytest.mark.unit + def test_the_message_names_what_is_missing_and_where_it_comes_from(self): + message = _deps.groundwater_deps_message(["mf2005"]) + + assert "mf2005" in message + assert "conda-forge" in message + assert _deps._ENVIRONMENT_YML_URL in message + + @pytest.mark.unit + def test_the_message_defaults_to_every_groundwater_dependency(self): + assert _deps.groundwater_deps_message() == _deps.groundwater_deps_message( + ["pcraster._pcraster_modflow", "mf2005"] + ) + + @pytest.mark.unit + def test_the_prefix_directory_is_put_on_path_once( + self, monkeypatch, no_executable_on_path, caplog + ): + """The extension launches ``mf2005`` from PATH, and spawned workers inherit PATH.""" + directory = no_executable_on_path / "bin" + directory.mkdir() + (directory / "mf2005").write_bytes(b"") + monkeypatch.setenv("PATH", "elsewhere") + + with caplog.at_level("INFO", logger=_deps.__name__): + first = _deps.ensure_mf2005_on_path() + second = _deps.ensure_mf2005_on_path() + + assert first == second == directory / "mf2005" + entries = _deps.os.environ["PATH"].split(_deps.os.pathsep) + assert entries == [str(directory), "elsewhere"] + assert len([r for r in caplog.records if str(directory) in r.getMessage()]) == 1 + + @pytest.mark.unit + def test_path_is_left_alone_when_the_executable_is_on_it(self, monkeypatch, tmp_path): + monkeypatch.setattr(_deps.shutil, "which", lambda name: str(tmp_path / "mf2005")) + monkeypatch.setenv("PATH", "elsewhere") + + assert _deps.ensure_mf2005_on_path() == tmp_path / "mf2005" + assert _deps.os.environ["PATH"] == "elsewhere" + + @pytest.mark.unit + def test_nothing_is_put_on_path_without_an_executable(self, monkeypatch, no_executable_on_path): + monkeypatch.setenv("PATH", "elsewhere") + + assert _deps.ensure_mf2005_on_path() is None + assert _deps.os.environ["PATH"] == "elsewhere" diff --git a/tests/unit/validation/test_modflow_inputs.py b/tests/unit/validation/test_modflow_inputs.py new file mode 100644 index 0000000..b0a98ec --- /dev/null +++ b/tests/unit/validation/test_modflow_inputs.py @@ -0,0 +1,746 @@ +"""Checks of the MODFLOW inputs and runtime run before a coupled simulation. + +The rasters are real PCRaster maps on the 3x3 synthetic grid: pyfakefs cannot +back PCRaster or GDAL I/O. +""" + +import math +import os +from pathlib import Path + +import numpy as np +import pytest + +from rubem import _deps +from rubem.configuration._problems import ConfigurationError +from rubem.configuration.input_raster_files import InputRasterFiles +from rubem.configuration.input_table_files import InputTableFiles +from rubem.configuration.model_configuration import ModelConfiguration +from rubem.configuration.model_configuration_file import ModelConfigurationFile +from rubem.configuration.model_configuration_file_v1 import ModelConfigurationFileV1 +from rubem.configuration.modflow_configuration import ModflowSettings +from rubem.validation import modflow_inputs +from rubem.validation.modflow_inputs import check_modflow_inputs +from tests.helpers.synthetic import ( + CELL_SIZE, + MODFLOW_HEAD, + MODFLOW_LAYER_THICKNESS, + MODFLOW_TOP, + NORTH, + WEST, + write_grid_map, + write_modflow_inputs, + write_synthetic_dataset, +) + +NAN = math.nan +# Row-major 3x3 grids: the cell at row 2, column 3 is index 5. +ROW_2_COLUMN_3 = 5 + + +@pytest.fixture(name="config") +def config_fixture(tmp_path): + """The synthetic dataset with a valid three-layer MODFLOW section.""" + config = write_synthetic_dataset(str(tmp_path)) + config["MODFLOW"] = write_modflow_inputs(config) + return config + + +def raster_files(config): + rasters = config["RASTERS"] + return InputRasterFiles( + dem=rasters["dem"], + clone=rasters["clone"], + ndvi_max=rasters["ndvi_max"], + ndvi_min=rasters["ndvi_min"], + soil=rasters["soil"], + ldd=rasters["ldd"], + sample_locations=rasters["samples"], + validate_input=False, + ) + + +def table_files(config): + tables = dict(config["TABLES"]) + tables["rainy_days"] = tables.pop("rainydays") + tables["kc_min"] = tables.pop("k_c_min") + tables["kc_max"] = tables.pop("k_c_max") + return InputTableFiles(**tables, validate_input=False) + + +def check(config, validate_input=True): + settings = ModflowSettings.model_validate(config["MODFLOW"]) + return check_modflow_inputs( + settings, raster_files(config), validate_input, tables=table_files(config) + ) + + +def blocking(problems): + return [problem for problem in problems if problem.blocking] + + +def sibling(config, name): + """A path in the directory of the MODFLOW inputs.""" + return os.path.join(os.path.dirname(config["MODFLOW"]["top"]), name) + + +def grid(values, at=None, value=None): + """Nine values; ``values`` broadcast, with ``value`` placed at index ``at``.""" + array = np.full(9, values, dtype=float) + if at is not None: + array[at] = value + return array + + +def enable_root_depth(config, rows): + table = sibling(config, "dpz_min.txt") + Path(table).write_text("".join(f"{key} {value}\n" for key, value in rows), encoding="utf8") + config["MODFLOW"]["coupling"] = { + "dynamic_root_depth": {"enabled": True, "minimum_depth_table": table} + } + return table + + +class TestValidInputs: + @pytest.mark.unit + @pytest.mark.parametrize("validate_input", [True, False]) + def test_the_synthetic_inputs_have_no_problem(self, config, validate_input): + assert check(config, validate_input) == [] + + +class TestFiles: + @pytest.mark.unit + @pytest.mark.parametrize("validate_input", [True, False]) + def test_a_missing_file_is_blocking_with_or_without_validation(self, config, validate_input): + missing = sibling(config, "absent.map") + config["MODFLOW"]["layers"][1]["bottom"] = missing + + problems = check(config, validate_input) + + assert len(problems) == 1 + assert problems[0].blocking + assert problems[0].description == "MODFLOW input file does not exist." + assert Path(problems[0].file) == Path(missing) + + @pytest.mark.unit + def test_a_missing_lookup_table_is_reported(self, config): + table = sibling(config, "absent.tbl") + config["MODFLOW"]["layers"][0]["horizontal_conductivity"]["table"] = table + + problems = check(config) + + assert [Path(p.file) for p in blocking(problems)] == [Path(table)] + + @pytest.mark.unit + def test_an_empty_file_is_blocking(self, config): + empty = sibling(config, "empty.map") + Path(empty).write_bytes(b"") + config["MODFLOW"]["top"] = empty + + problems = check(config, validate_input=False) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == "MODFLOW input file is empty." + assert Path(problems[0].file) == Path(empty) + + @pytest.mark.unit + def test_a_file_shared_by_several_layers_is_reported_once(self, config): + boundary = sibling(config, "absent_bound.map") + for item in config["MODFLOW"]["layers"]: + item["boundary"] = boundary + + assert len(check(config)) == 1 + + @pytest.mark.unit + def test_the_files_of_what_the_run_does_not_read_are_not_checked(self, config): + """A disabled package may keep entries whose files are gone.""" + absent = sibling(config, "absent.map") + config["MODFLOW"]["drain"] = { + "enabled": False, + "entries": [{"layers": [1], "elevation": absent, "conductance": absent}], + } + config["MODFLOW"]["wetting"] = {"enabled": False, "map": absent} + config["MODFLOW"]["coupling"] = { + "dynamic_root_depth": {"enabled": False, "minimum_depth_table": absent} + } + + assert check(config) == [] + + @pytest.mark.unit + def test_the_content_is_not_read_when_a_file_is_missing(self, config): + config["MODFLOW"]["top"] = sibling(config, "absent.map") + write_grid_map(config["MODFLOW"]["layers"][0]["boundary"], 7, nominal=True) + + problems = check(config) + + assert [p.description for p in problems] == ["MODFLOW input file does not exist."] + + +class TestRuntime: + @pytest.mark.unit + @pytest.mark.parametrize("validate_input", [True, False]) + def test_a_missing_runtime_is_blocking_with_or_without_validation( + self, config, monkeypatch, validate_input + ): + monkeypatch.setattr(modflow_inputs, "missing_groundwater_deps", lambda: ["mf2005"]) + + problems = check(config, validate_input) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == "MODFLOW runtime is not available." + assert problems[0].reason == _deps.groundwater_deps_message(["mf2005"]) + + @pytest.mark.unit + def test_the_content_is_not_read_without_pcraster(self, config, monkeypatch): + monkeypatch.setattr( + modflow_inputs, "missing_groundwater_deps", lambda: ["pcraster._pcraster_modflow"] + ) + monkeypatch.setattr(modflow_inputs, "missing_runtime_deps", lambda: ["pcraster"]) + write_grid_map(config["MODFLOW"]["layers"][0]["boundary"], 7, nominal=True) + + problems = check(config) + + assert [p.description for p in problems] == ["MODFLOW runtime is not available."] + + +class TestRasters: + @pytest.mark.unit + def test_the_content_is_checked_only_with_validation(self, config): + write_grid_map(config["MODFLOW"]["layers"][0]["boundary"], 7, nominal=True) + + assert check(config, validate_input=False) == [] + assert blocking(check(config, validate_input=True)) + + @pytest.mark.unit + def test_a_raster_of_another_geometry_is_blocking(self, config): + """PCRaster reads a map of another size on the clone without complaint.""" + import pcraster as pcr + + small = sibling(config, "small.map") + pcr.setclone(2, 2, CELL_SIZE, WEST, NORTH) + pcr.report(pcr.numpy2pcr(pcr.Scalar, np.full((2, 2), 50.0, np.float32), -9999.0), small) + config["MODFLOW"]["layers"][2]["bottom"] = small + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == "MODFLOW raster does not share the clone geometry." + assert Path(problems[0].file) == Path(small) + + @pytest.mark.unit + def test_an_unreadable_raster_is_blocking(self, config): + broken = sibling(config, "broken.map") + Path(broken).write_text("not a raster", encoding="utf8") + config["MODFLOW"]["top"] = broken + + problems = check(config) + + assert problems and all(p.blocking for p in problems) + assert Path(problems[0].file) == Path(broken) + assert "cannot be read" in problems[0].description + + +class TestBoundary: + @pytest.mark.unit + def test_a_boundary_value_other_than_minus_one_zero_or_one_is_blocking(self, config): + path = sibling(config, "bound2.map") + write_grid_map(path, grid(1, at=ROW_2_COLUMN_3, value=2), nominal=True) + config["MODFLOW"]["layers"][1]["boundary"] = path + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == "MODFLOW boundary of layer 2 (layer2) has invalid values." + assert "[2]" in problems[0].reason + assert Path(problems[0].file) == Path(path) + + @pytest.mark.unit + def test_constant_head_and_missing_cells_are_accepted(self, config): + path = sibling(config, "bound_ch.map") + values = grid(1, at=0, value=-1) + values[8] = NAN + write_grid_map(path, values, nominal=True) + config["MODFLOW"]["layers"][0]["boundary"] = path + + assert check(config) == [] + + +class TestGeometry: + @pytest.mark.unit + def test_a_missing_top_on_an_active_column_is_blocking(self, config): + write_grid_map(config["MODFLOW"]["top"], grid(MODFLOW_TOP, at=ROW_2_COLUMN_3, value=NAN)) + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == "MODFLOW model top has missing values on active columns." + assert "row 2, column 3" in problems[0].reason + + @pytest.mark.unit + def test_a_bottom_that_is_not_below_the_layer_above_is_blocking(self, config): + upper = MODFLOW_TOP - MODFLOW_LAYER_THICKNESS + write_grid_map( + config["MODFLOW"]["layers"][1]["bottom"], + grid(upper - MODFLOW_LAYER_THICKNESS, at=ROW_2_COLUMN_3, value=upper), + ) + + problems = check(config) + + geometry = [p for p in problems if "not below" in p.description] + assert len(geometry) == 1 and geometry[0].blocking + assert geometry[0].description == ( + "MODFLOW bottom of layer 2 (layer2) is not below the top of the layer." + ) + assert "row 2, column 3" in geometry[0].reason + assert "1 cell" in geometry[0].reason + + @pytest.mark.unit + def test_a_bottom_above_the_model_top_is_blocking(self, config): + write_grid_map(config["MODFLOW"]["layers"][0]["bottom"], grid(80.0, at=0, value=120.0)) + + problems = check(config) + + assert any( + p.blocking and p.description.startswith("MODFLOW bottom of layer 1 (layer1)") + for p in problems + ) + + @pytest.mark.unit + def test_columns_inactive_in_every_layer_are_not_checked(self, config): + """The coupling fills those columns with synthetic elevations.""" + boundary = sibling(config, "bound_hole.map") + write_grid_map(boundary, grid(1, at=ROW_2_COLUMN_3, value=0), nominal=True) + for item in config["MODFLOW"]["layers"]: + item["boundary"] = boundary + write_grid_map(config["MODFLOW"]["top"], grid(MODFLOW_TOP, at=ROW_2_COLUMN_3, value=NAN)) + write_grid_map( + config["MODFLOW"]["layers"][2]["bottom"], grid(40.0, at=ROW_2_COLUMN_3, value=500.0) + ) + + assert check(config) == [] + + +class TestLayerValues: + @pytest.mark.unit + def test_a_missing_initial_head_on_an_active_cell_is_blocking(self, config): + path = sibling(config, "head_hole.map") + write_grid_map(path, grid(MODFLOW_HEAD, at=0, value=NAN)) + config["MODFLOW"]["layers"][2]["initial_head"] = path + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == ( + "MODFLOW initial head of layer 3 (layer3) has missing values on active cells." + ) + assert "row 1, column 1" in problems[0].reason + + @pytest.mark.unit + def test_a_missing_value_on_an_inactive_cell_is_accepted(self, config): + boundary = sibling(config, "bound_hole.map") + write_grid_map(boundary, grid(1, at=0, value=0), nominal=True) + config["MODFLOW"]["layers"][2]["boundary"] = boundary + path = sibling(config, "head_hole.map") + write_grid_map(path, grid(MODFLOW_HEAD, at=0, value=NAN)) + config["MODFLOW"]["layers"][2]["initial_head"] = path + + assert check(config) == [] + + @pytest.mark.unit + @pytest.mark.parametrize( + "key, label", + [ + ("vertical_conductivity", "vertical conductivity"), + ("horizontal_conductivity", "horizontal conductivity"), + ("specific_storage", "specific storage"), + ("specific_yield", "specific yield"), + ], + ) + def test_a_missing_property_on_an_active_cell_is_blocking(self, config, key, label): + path = sibling(config, f"{key}.map") + write_grid_map(path, grid(0.1, at=ROW_2_COLUMN_3, value=NAN)) + config["MODFLOW"]["layers"][1][key] = path + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == ( + f"MODFLOW {label} of layer 2 (layer2) has missing values on active cells." + ) + + @pytest.mark.unit + def test_a_head_below_the_bottom_in_every_active_cell_is_blocking(self, config): + path = sibling(config, "head_low.map") + write_grid_map(path, 10.0) + config["MODFLOW"]["layers"][1]["initial_head"] = path + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == ( + "MODFLOW initial head of layer 2 (layer2) is below the layer bottom in every " + "active cell." + ) + + @pytest.mark.unit + def test_a_head_below_the_bottom_in_some_active_cells_is_a_warning(self, config): + path = sibling(config, "head_low.map") + values = grid(MODFLOW_HEAD, at=0, value=10.0) + values[1] = 10.0 + write_grid_map(path, values) + config["MODFLOW"]["layers"][1]["initial_head"] = path + + problems = check(config) + + assert len(problems) == 1 and not problems[0].blocking + assert problems[0].description == ( + "MODFLOW initial head of layer 2 (layer2) is below the layer bottom in some " + "active cells." + ) + assert problems[0].reason.startswith("2 of 9 active cells") + + @pytest.mark.unit + def test_a_no_data_sentinel_in_an_active_cell_is_blocking(self, config): + path = sibling(config, "head_sentinel.map") + write_grid_map(path, grid(MODFLOW_HEAD, at=ROW_2_COLUMN_3, value=-888.0)) + config["MODFLOW"]["layers"][1]["initial_head"] = path + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == ( + "MODFLOW initial head of layer 2 (layer2) holds a no-data sentinel in active cells." + ) + assert problems[0].reason == ( + "1 of 9 active cells hold the value -888, a MODFLOW no-data marker, not a head; " + "the first at row 2, column 3." + ) + assert Path(problems[0].file) == Path(path) + + @pytest.mark.unit + def test_a_no_data_sentinel_in_an_inactive_cell_is_accepted(self, config): + boundary = sibling(config, "bound_hole.map") + write_grid_map(boundary, grid(1, at=ROW_2_COLUMN_3, value=0), nominal=True) + path = sibling(config, "head_sentinel.map") + write_grid_map(path, grid(MODFLOW_HEAD, at=ROW_2_COLUMN_3, value=-999.99)) + config["MODFLOW"]["layers"][1]["boundary"] = boundary + config["MODFLOW"]["layers"][1]["initial_head"] = path + + assert check(config) == [] + + +class TestConductivityClasses: + @pytest.mark.unit + def test_a_class_absent_from_the_table_is_blocking(self, config): + lookup = config["MODFLOW"]["layers"][0]["horizontal_conductivity"] + Path(lookup["table"]).write_text("1 0.5\n", encoding="utf8") + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == ( + "MODFLOW horizontal conductivity table of layer 1 (layer1) does not cover the " + "classes of its map." + ) + assert "[2]" in problems[0].reason + assert Path(problems[0].file) == Path(lookup["table"]) + + @pytest.mark.unit + @pytest.mark.parametrize("value", ["0", "-0.1", "nan"]) + def test_a_class_without_a_positive_value_is_blocking(self, config, value): + lookup = config["MODFLOW"]["layers"][0]["horizontal_conductivity"] + Path(lookup["table"]).write_text(f"1 0.5\n2 {value}\n", encoding="utf8") + + problems = blocking(check(config)) + + assert len(problems) == 1 + assert Path(problems[0].file) == Path(lookup["table"]) + + @pytest.mark.unit + def test_an_interval_key_covers_its_classes(self, config): + lookup = config["MODFLOW"]["layers"][0]["horizontal_conductivity"] + Path(lookup["table"]).write_text("[1,2] 0.5\n", encoding="utf8") + + assert check(config) == [] + + @pytest.mark.unit + def test_classes_of_inactive_cells_need_no_value(self, config): + boundary = sibling(config, "bound_top_row_off.map") + write_grid_map(boundary, [0, 0, 0, 1, 1, 1, 1, 1, 1], nominal=True) + config["MODFLOW"]["layers"][0]["boundary"] = boundary + lookup = config["MODFLOW"]["layers"][0]["horizontal_conductivity"] + Path(lookup["table"]).write_text("2 0.1\n", encoding="utf8") + + assert check(config) == [] + + @pytest.mark.unit + def test_an_unreadable_table_is_blocking(self, config): + lookup = config["MODFLOW"]["layers"][0]["horizontal_conductivity"] + Path(lookup["table"]).write_text("1 0.5 extra\n", encoding="utf8") + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert "cannot be read" in problems[0].description + + @pytest.mark.unit + def test_a_class_map_that_is_not_nominal_is_blocking(self, config): + lookup = config["MODFLOW"]["layers"][0]["horizontal_conductivity"] + write_grid_map(lookup["map"], 1.0) + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert "nominal" in problems[0].description + + @pytest.mark.unit + def test_a_missing_class_on_an_active_cell_is_blocking(self, config): + lookup = config["MODFLOW"]["layers"][0]["horizontal_conductivity"] + write_grid_map(lookup["map"], grid(2, at=0, value=NAN), nominal=True) + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert "row 1, column 1" in problems[0].reason + + +class TestActiveCellGuard: + @pytest.mark.unit + def test_a_package_layer_without_a_positive_conductance_is_blocking(self, config): + entry = config["MODFLOW"]["ghb"]["entries"][0] + path = sibling(config, "ghb_cond_split.map") + # Conductance only on row 1; layer 2 is inactive there. + write_grid_map(path, [1, 1, 1, 0, 0, 0, 0, 0, 0]) + entry["conductance"] = path + boundary = sibling(config, "bound_top_row_off.map") + write_grid_map(boundary, [0, 0, 0, 1, 1, 1, 1, 1, 1], nominal=True) + config["MODFLOW"]["layers"][1]["boundary"] = boundary + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == "MODFLOW ghb package has no cell in layer 2 (layer2)." + assert "aborts the process" in problems[0].implication + + @pytest.mark.unit + def test_river_cells_are_selected_by_the_mask(self, config): + write_grid_map(config["MODFLOW"]["river"]["entries"][0]["mask"], 0) + + problems = check(config) + + assert [p.description for p in problems] == [ + "MODFLOW river package has no cell in layer 1 (layer1)." + ] + + @pytest.mark.unit + def test_river_cells_outside_the_boundary_of_the_layer_do_not_count(self, config): + boundary = sibling(config, "bound_middle_off.map") + write_grid_map(boundary, [1, 0, 1, 1, 0, 1, 1, 0, 1], nominal=True) + config["MODFLOW"]["layers"][0]["boundary"] = boundary + + problems = check(config) + + assert [p.description for p in problems] == [ + "MODFLOW river package has no cell in layer 1 (layer1)." + ] + + @pytest.mark.unit + def test_an_enabled_drain_layer_without_cells_is_blocking(self, config): + config["MODFLOW"]["drain"] = { + "enabled": True, + "entries": [ + { + "layers": [3], + "elevation": config["MODFLOW"]["top"], + "conductance": write_grid_map(sibling(config, "drn_cond.map"), 0), + } + ], + } + + problems = check(config) + + assert [p.description for p in problems] == [ + "MODFLOW drain package has no cell in layer 3 (layer3)." + ] + + @pytest.mark.unit + def test_a_missing_stress_value_on_a_package_cell_is_blocking(self, config): + """GHB cells are the left column; a missing head elsewhere is harmless.""" + head = config["MODFLOW"]["ghb"]["entries"][0]["head"] + write_grid_map(head, grid(MODFLOW_HEAD, at=1, value=NAN)) + assert check(config) == [] + + write_grid_map(head, grid(MODFLOW_HEAD, at=3, value=NAN)) + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == ( + "MODFLOW ghb head has missing values on the cells of the package." + ) + assert "row 2, column 1" in problems[0].reason + + +class TestRiverbed: + @pytest.mark.unit + def test_riverbeds_outside_the_layer_are_counted_without_blocking(self, config): + # River cells are the middle column; layer 1 spans [80, 100] m. + bottom = config["MODFLOW"]["river"]["entries"][0]["bottom"] + write_grid_map(bottom, [0, 70.0, 0, 0, 90.0, 0, 0, 105.0, 0]) + + problems = check(config) + + assert len(problems) == 1 and not problems[0].blocking + assert problems[0].reason.startswith( + "2 of 3 river cells of layer 1 have their bed outside the layer's elevation interval" + ) + assert "1 below the layer bottom" in problems[0].reason + assert "1 above the layer top" in problems[0].reason + assert Path(problems[0].file) == Path(bottom) + + @pytest.mark.unit + def test_the_interval_of_a_lower_layer_runs_between_its_bottom_and_the_bottom_above( + self, config + ): + config["MODFLOW"]["river"]["entries"][0]["layers"] = [2] + + problems = check(config) + + assert len(problems) == 1 and not problems[0].blocking + assert problems[0].reason.startswith( + "3 of 3 river cells of layer 2 have their bed outside the layer's elevation interval" + ) + + @pytest.mark.unit + def test_a_bed_on_the_limits_of_the_interval_is_inside(self, config): + bottom = config["MODFLOW"]["river"]["entries"][0]["bottom"] + write_grid_map(bottom, [0, 80.0, 0, 0, 90.0, 0, 0, 100.0, 0]) + + assert check(config) == [] + + +class TestMinimumRootDepth: + @pytest.mark.unit + def test_a_valid_table_has_no_problem(self, config): + enable_root_depth(config, [(1, 50.0)]) + + assert check(config) == [] + + @pytest.mark.unit + def test_a_soil_class_absent_from_the_table_is_blocking(self, config): + table = enable_root_depth(config, [(2, 50.0)]) + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == ( + "Minimum root depth (Dpz_min) lookup table does not cover the soil classes." + ) + assert Path(problems[0].file) == Path(table) + + @pytest.mark.unit + @pytest.mark.parametrize( + "value, description", + [ + (0.0, "Minimum root depth (Dpz_min) lookup table has non-positive values."), + (-5.0, "Minimum root depth (Dpz_min) lookup table has non-positive values."), + (200.0, "Minimum root depth (Dpz_min) exceeds the rootzone depth (Zr)."), + ], + ) + def test_a_value_outside_zero_and_the_rootzone_depth_is_blocking( + self, config, value, description + ): + """The synthetic rootzone depth (Zr) of soil class 1 is 150.39.""" + enable_root_depth(config, [(1, value)]) + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert problems[0].description == description + assert "soil class 1" in problems[0].reason + + @pytest.mark.unit + def test_an_unreadable_table_is_blocking(self, config): + table = enable_root_depth(config, [(1, "x")]) + + problems = check(config) + + assert len(problems) == 1 and problems[0].blocking + assert "cannot be read" in problems[0].description + assert Path(problems[0].file) == Path(table) + + +class TestModelConfiguration: + @pytest.fixture(name="plain") + def plain_fixture(self, tmp_path): + return write_synthetic_dataset(str(tmp_path)) + + @staticmethod + def formats(config): + legacy = ModelConfigurationFile.model_validate(config) + return (config, ModelConfigurationFileV1.from_legacy(legacy).to_dict()) + + @pytest.mark.unit + @pytest.mark.parametrize("validate_input", [True, False]) + def test_an_absent_section_runs_no_check_and_adds_no_problem( + self, plain, monkeypatch, validate_input + ): + """The problems of a configuration without the section are the ones of main.""" + baseline = [ + ModelConfiguration(source, validate_input=validate_input).problems + for source in self.formats(plain) + ] + + def refuse(*args, **kwargs): + raise AssertionError("check_modflow_inputs ran without an enabled section") + + monkeypatch.setattr("rubem.configuration.model_configuration.check_modflow_inputs", refuse) + for source, expected in zip(self.formats(plain), baseline, strict=True): + loaded = ModelConfiguration(source, validate_input=validate_input) + assert loaded.problems == expected + assert baseline == [[], []] + + @pytest.mark.unit + def test_a_disabled_section_runs_no_check(self, config, monkeypatch): + config["MODFLOW"]["enabled"] = False + config["MODFLOW"]["top"] = "absent.map" + + def refuse(*args, **kwargs): + raise AssertionError("check_modflow_inputs ran for a disabled section") + + monkeypatch.setattr("rubem.configuration.model_configuration.check_modflow_inputs", refuse) + for source in self.formats(config): + loaded = ModelConfiguration(source) + assert [p.description for p in loaded.problems] == ["MODFLOW section is ignored."] + + @pytest.mark.unit + def test_valid_inputs_load_without_problems(self, config): + for source in self.formats(config): + assert ModelConfiguration(source).problems == [] + + @pytest.mark.unit + @pytest.mark.parametrize("validate_input", [True, False]) + def test_an_enabled_section_with_missing_files_is_blocking(self, config, validate_input): + config["MODFLOW"]["top"] = sibling(config, "absent.map") + + for source in self.formats(config): + with pytest.raises(ConfigurationError, match="MODFLOW input file does not exist"): + ModelConfiguration(source, validate_input=validate_input) + + @pytest.mark.unit + def test_the_content_is_checked_only_with_validation(self, config): + write_grid_map(config["MODFLOW"]["layers"][0]["boundary"], 7, nominal=True) + + for source in self.formats(config): + assert ModelConfiguration(source, validate_input=False).problems == [] + with pytest.raises(ConfigurationError, match="MODFLOW boundary of layer 1"): + ModelConfiguration(source) + + @pytest.mark.unit + def test_the_minimum_root_depth_is_compared_with_the_configured_rootzone_depth(self, config): + enable_root_depth(config, [(1, 200.0)]) + + for source in self.formats(config): + loaded = ModelConfiguration(source, allow_blocking_problems=True) + assert [p.description for p in blocking(loaded.problems)] == [ + "Minimum root depth (Dpz_min) exceeds the rootzone depth (Zr)." + ]