From e7995383036e969594cde31698654cb59ae73318 Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Fri, 9 Oct 2026 12:09:01 +0000 Subject: [PATCH 01/10] result as property --- examples/tutorial.py | 3 +- src/floatmatcher/floatmatcher.py | 92 +++++++++++++++++++++++++---- src/floatmatcher/matchup_results.py | 4 +- tests/test_floatmatcher.py | 39 +++++++++--- 4 files changed, 118 insertions(+), 20 deletions(-) diff --git a/examples/tutorial.py b/examples/tutorial.py index f57e8e0..13a12ef 100644 --- a/examples/tutorial.py +++ b/examples/tutorial.py @@ -73,10 +73,11 @@ # my_fm.status() # TODO -result = my_fm.match() # or resultats = my_fm.match() +# my_fm.match() # or resultats = my_fm.match() # Compte rendu / visualisation etc.. my_fm.result + # my_fm.result_summary() # my_fm.result_viz(header=Npoints) diff --git a/src/floatmatcher/floatmatcher.py b/src/floatmatcher/floatmatcher.py index 27fdf54..95b9484 100644 --- a/src/floatmatcher/floatmatcher.py +++ b/src/floatmatcher/floatmatcher.py @@ -3,6 +3,8 @@ from typing import Any import numpy as np +import pandas as pd +import xarray as xr from numpy.typing import NDArray from .methods import Method, available_methods @@ -12,11 +14,35 @@ from .product import Product, available_products class FloatMatcher: - """FloatMatcher gather all Points Pointset(), variables needed, setup Product(). - It prepares all elements to call the method in .match() function. - All constraints and parametrization lives in .match() - """ + """A `FloatMatcher` starts empty and is filled with the ``set_*`` methods: + the points to match, the product to match them with, and the matching + method. The function ``match()`` then runs the matchup. + Results are stored in result object + + Attributes + ---------- + points : PointSet or None + Positions (lon, lat, time) to match. + product : Product or None + Gridded product the points are matched with. + method : Method or None + Matching method and its parameters. + result : MatchupResult or None + Result of the last ``match()`` call. + + Examples + -------- + >>> fm = FloatMatcher() + >>> fm.set_points_from_arrays(lon, lat, time) + >>> fm.set_product("era5", "local", "/data/era5", ["sst"]) + >>> fm.set_method("nearest", radius=50, time_gap=np.timedelta64(3, "h")) + >>> result = fm.match() + """ + + + + def __init__(self, points: PointSet | None = None, product: Product | None = None, @@ -28,13 +54,14 @@ def __init__(self, self.product = product self.variables = variables self.method = method - self.result: MatchupResult | None = None + self._result: MatchupResult | None = None + def __repr__(self,) -> str: return (f"points:{self.points} \nproduct:{self.product} " f"\nvariables:{self.variables} \nmethod: {self.method}" - f"\nresults: {self.result}") + f"\nresults: {self._result}") @@ -71,9 +98,14 @@ def set_method(self, type: str, **params: Any) -> None: self.method = available_methods[type](**params) + @property + def result(self) -> MatchupResult: + if self._result is None : + raise AttributeError("no result yet, call match()") + return self._result - def match(self) -> MatchupResult: + def match(self) -> None: if self.points is None: raise ValueError("match(): no points set, call set_points_from_arrays(...)") if self.product is None: @@ -84,8 +116,48 @@ def match(self) -> MatchupResult: # Get files opened as a mfDataset in product.src_dataset self.product.open(points=self.points) - self.result = self.method.apply(self.product.selected_dataset, self.points) - return self.result + self._result = self.method.apply(self.product.selected_dataset, self.points) - + + def to_dataset(self) -> xr.Dataset: + """Add the matched values to the Dataset the points come from. + + Values are added as ``_coloc``. The source Dataset is not modified. + + Returns + ------- + xarray.Dataset + + Raises + ------ + ValueError + No result yet, or points not loaded from a Dataset. + TypeError + Points loaded from a DataFrame, use ``to_dataframe()``. + """ + if self._result is None or self.points is None: + raise ValueError("no result yet or points are None, try call match()") + + return self._result._to_dataset(self.points) + + def to_dataframe(self) -> pd.DataFrame: + """Add the matched values to the DataFrame the points come from. + + Values are added as ``_coloc`` columns. The source DataFrame is not modified. + + Returns + ------- + pandas.DataFrame + + Raises + ------ + ValueError + No result yet, or points not loaded from a DataFrame. + TypeError + Points loaded from a Dataset, use ``to_dataset()``. + """ + if self._result is None or self.points is None: + raise ValueError("no result yet or points are None, try call match()") + + return self._result._to_dataframe(self.points) diff --git a/src/floatmatcher/matchup_results.py b/src/floatmatcher/matchup_results.py index a1f637c..e6e2136 100644 --- a/src/floatmatcher/matchup_results.py +++ b/src/floatmatcher/matchup_results.py @@ -38,7 +38,7 @@ def _to_dataset(self, points: PointSet) -> xr.Dataset: if not isinstance(ds, xr.Dataset): raise TypeError( f"Cannot reinject into a Dataset: the origin is a {type(ds).__name__}. " - "Use to_dataframe()." + "Use FloatMatcher.to_dataframe()." ) out = ds.copy() @@ -61,7 +61,7 @@ def _to_dataframe(self, points: PointSet) -> pd.DataFrame: if not isinstance(df, pd.DataFrame): raise TypeError( f"Cannot reinject into a DataFrame: the origin is a {type(df).__name__}. " - "Use to_dataset()." + "Use FloatMatcher.to_dataset()." ) out = df.copy() diff --git a/tests/test_floatmatcher.py b/tests/test_floatmatcher.py index 31fa651..51c59eb 100644 --- a/tests/test_floatmatcher.py +++ b/tests/test_floatmatcher.py @@ -19,7 +19,9 @@ def test_match_nearest_get_good_node(era5_files): ) product = ERA5Product("local", era5_files, ["sst"]) - res = FloatMatcher(points=points, product=product, method=NearestNeighbor()).match() + fm = FloatMatcher(points=points, product=product, method=NearestNeighbor()) + fm.match() + res = fm.result assert res.valid.all() assert set(res.values) == {"sst"} # only sst has been taken into account @@ -42,7 +44,8 @@ def test_match_nearest_rejects_out_of_range_points(era5_files): orch = FloatMatcher(points=points, product=product) orch.set_method("nearest") - res = orch.match() + orch.match() + res = orch.result assert res.valid.tolist() == [True, False, False] npt.assert_allclose(res.values["sst"][:1], [121.03], atol=1e-4) @@ -52,12 +55,14 @@ def test_match_nearest_rejects_out_of_range_points(era5_files): # p1 was out on DISTANCE only orch.set_method("nearest", radius=100000) - loose_dist = orch.match() + orch.match() + loose_dist = orch.result assert loose_dist.valid.tolist() == [True, True, False] # p2 was out on TIME only orch.set_method("nearest", time_gap=np.timedelta64(300, "D")) - loose_time = orch.match() + orch.match() + loose_time = orch.result assert loose_time.valid.tolist() == [True, False, True] @@ -68,7 +73,9 @@ def test_match_nearest_over_two_variable(era5_files): ) product = ERA5Product("local", era5_files, ["sst", "t2m"]) - res = FloatMatcher(points=points, product=product, method=NearestNeighbor()).match() + fm = FloatMatcher(points=points, product=product, method=NearestNeighbor()) + fm.match() + res = fm.result assert set(res.values) == {"sst", "t2m"} npt.assert_allclose(res.values["sst"], [121.03], atol=1e-4) @@ -136,11 +143,11 @@ def test_match_through_the_builder(era5_files): fm.set_product("era5", "local", era5_files, ["sst"]) fm.set_method("nearest") - res = fm.match() + fm.match() + res = fm.result assert res.valid.tolist() == [True] npt.assert_allclose(res.values["sst"], [121.03], atol=1e-4) - assert fm.result is res # ---------- match : missing inputs ---------- @@ -164,3 +171,21 @@ def test_match_without_method_raises(era5_files): fm = FloatMatcher(points=_points(), product=ERA5Product("local", era5_files, ["sst"])) with pytest.raises(ValueError, match="no method set"): fm.match() + + +# ---------- result property ---------- + +def test_result_before_match_raises(): + fm = FloatMatcher() + + with pytest.raises(AttributeError, match="no result yet"): + fm.result + assert not hasattr(fm, "result") # attribute-like: hasattr answers instead of raising + + +def test_result_is_read_only(): + """the result only comes from match()""" + fm = FloatMatcher() + + with pytest.raises(AttributeError): + fm.result = None From 2a9194bbaae4be3dba5a42acef40673d4f702ec5 Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Fri, 9 Oct 2026 12:31:38 +0000 Subject: [PATCH 02/10] add reinject method (mask to_dataframe or to_dataset) add set_points generic + tests --- examples/tutorial.ipynb | 4 +- examples/tutorial.py | 4 +- src/floatmatcher/floatmatcher.py | 144 ++++++++++++++++++---- tests/test_floatmatcher.py | 197 ++++++++++++++++++++++++++++++- 4 files changed, 320 insertions(+), 29 deletions(-) diff --git a/examples/tutorial.ipynb b/examples/tutorial.ipynb index 8ea4f58..3ca9b33 100644 --- a/examples/tutorial.ipynb +++ b/examples/tutorial.ipynb @@ -116,7 +116,7 @@ "# TODO : set_points_from dataframe/dataset\n", "\n", "# 1.1 example\n", - "my_fm.set_points_from_arrays(lon, lat, time)\n", + "my_fm.set_points(lon, lat, time)\n", "my_fm" ] }, @@ -327,7 +327,7 @@ "lon, lat, time = mocking.random_positions(50)\n", "\n", "my_fm = fm.FloatMatcher()\n", - "my_fm.set_points_from_arrays(lon, lat, time)\n", + "my_fm.set_points(lon, lat, time)\n", "\n", "# set_local_product not implemented yet: product and variables set directly\n", "my_fm.product = ERA5Product.from_local(path=\"/runtime/data/era5_daily/2018/01\")\n", diff --git a/examples/tutorial.py b/examples/tutorial.py index 13a12ef..54152ca 100644 --- a/examples/tutorial.py +++ b/examples/tutorial.py @@ -37,7 +37,7 @@ # TODO : set_points_from dataframe/dataset # 1.1 example -my_fm.set_points_from_arrays(lon, lat, time) +my_fm.set_points(lon, lat, time) @@ -73,7 +73,7 @@ # my_fm.status() # TODO -# my_fm.match() # or resultats = my_fm.match() +my_fm.match() # Compte rendu / visualisation etc.. my_fm.result diff --git a/src/floatmatcher/floatmatcher.py b/src/floatmatcher/floatmatcher.py index 95b9484..4d0c8c3 100644 --- a/src/floatmatcher/floatmatcher.py +++ b/src/floatmatcher/floatmatcher.py @@ -2,10 +2,8 @@ from typing import Any -import numpy as np import pandas as pd import xarray as xr -from numpy.typing import NDArray from .methods import Method, available_methods # from .interpolation import Interpolation @@ -14,35 +12,32 @@ from .product import Product, available_products class FloatMatcher: - """A `FloatMatcher` starts empty and is filled with the ``set_*`` methods: - the points to match, the product to match them with, and the matching - method. The function ``match()`` then runs the matchup. - Results are stored in result object - + """Match positions with a gridded product. + + Fill it with the ``set_*`` methods (points, product, method), then call ``match()`` + and read the result with ``result`` or ``reinject()``. Attributes ---------- points : PointSet or None Positions (lon, lat, time) to match. product : Product or None - Gridded product the points are matched with. + Product the points are matched with. method : Method or None Matching method and its parameters. - result : MatchupResult or None - Result of the last ``match()`` call. + result : MatchupResult + Result of the last ``match()``. Raises if there is none yet. Examples -------- >>> fm = FloatMatcher() - >>> fm.set_points_from_arrays(lon, lat, time) + >>> fm.set_points(lon, lat, time) >>> fm.set_product("era5", "local", "/data/era5", ["sst"]) >>> fm.set_method("nearest", radius=50, time_gap=np.timedelta64(3, "h")) - >>> result = fm.match() + >>> fm.match() + >>> fm.result """ - - - - + def __init__(self, points: PointSet | None = None, product: Product | None = None, @@ -65,11 +60,36 @@ def __repr__(self,) -> str: - def set_points_from_arrays(self, - lon: NDArray[np.float64], - lat: NDArray[np.float64], - time: NDArray[np.datetime64] | None = None) -> None: - self.points = PointSet(lon, lat, time) + def set_points(self, *data: Any, **names: str) -> None: + """Set the points to match. The loader is chosen from the type of the input. + + Parameters + ---------- + *data + ``lon, lat[, time]`` arrays, a :class:`pandas.DataFrame`, a + :class:`xarray.Dataset` or a :class:`PointSet`. + **names : str + ``lon``, ``lat``, ``time``: names of the columns or variables holding them. + Only for a DataFrame or a Dataset. + + Raises + ------ + TypeError + Unsupported input, or names given with arrays or a PointSet. + """ + if len(data) == 1 and not names and isinstance(data[0], PointSet): + self.points = data[0] + elif len(data) == 1 and isinstance(data[0], xr.Dataset): + self.points = PointSet.from_xrdataset(data[0], **names) + elif len(data) == 1 and isinstance(data[0], (pd.DataFrame, pd.Series)): + self.points = PointSet.from_dataframe(data[0], **names) + elif len(data) in (2, 3) and not names: + self.points = PointSet.from_arrays(*data) + else: + raise TypeError( + "set_points() expects lon, lat[, time] arrays, a DataFrame, a Dataset or a PointSet " + "(lon=, lat=, time= are column or variable names, only for a DataFrame or a Dataset)" + ) @@ -80,7 +100,29 @@ def set_product(self, selected_variables: list[str] | None, **src_params: Any ) -> None: + """Set the product to match with. + + Parameters + ---------- + type : str + ``"era5"`` or ``"lut"``. + source : str + Where the files come from. Only ``"local"`` for now. + path : str or list of str + Directory or files of the product. With a ``pattern``, the root of the tree. + selected_variables : list of str or None + Variables to keep. None keeps all. + **src_params + Parameters of the source. ``"local"``: ``pattern``, a path template like + ``"{year}/{month:02d}/era5_{year}{month:02d}{day:02d}.nc"``. + Raises + ------ + ValueError + Unknown product, or source not available for it. + TypeError + Unknown source parameter. + """ if type not in available_products: raise ValueError(f"unknown product {type!r}, available: {list(available_products)}") @@ -93,6 +135,23 @@ def set_product(self, def set_method(self, type: str, **params: Any) -> None: + """Set the matching method. + + Parameters + ---------- + type : str + ``"nearest"``. + **params + Parameters of the method. ``"nearest"``: ``radius`` (km), + ``time_gap`` (``numpy.timedelta64``), ``k_nearest``. + + Raises + ------ + ValueError + Unknown method. + TypeError + Unknown method parameter. + """ if type not in available_methods: raise ValueError(f"unknown method {type!r}, available: {list(available_methods)}") self.method = available_methods[type](**params) @@ -100,14 +159,28 @@ def set_method(self, type: str, **params: Any) -> None: @property def result(self) -> MatchupResult: + """Result of the last ``match()``. + + Raises + ------ + AttributeError + No result yet. + """ if self._result is None : raise AttributeError("no result yet, call match()") return self._result def match(self) -> None: + """Run the matchup. Read the result with ``result``. + + Raises + ------ + ValueError + Points, product or method not set. + """ if self.points is None: - raise ValueError("match(): no points set, call set_points_from_arrays(...)") + raise ValueError("match(): no points set, call set_points(...)") if self.product is None: raise ValueError("match(): no product set, call set_product(...)") if self.method is None: @@ -161,3 +234,30 @@ def to_dataframe(self) -> pd.DataFrame: raise ValueError("no result yet or points are None, try call match()") return self._result._to_dataframe(self.points) + + def reinject(self) -> xr.Dataset | pd.DataFrame: + """Add the matched values to the object the points come from. + + Same type as the input of ``set_points``: a Dataset gives a Dataset, a DataFrame + gives a DataFrame. Values are added as ``_coloc``. The source is not modified. + + Returns + ------- + xarray.Dataset or pandas.DataFrame + + Raises + ------ + ValueError + No result yet, or points not loaded from a Dataset or a DataFrame. + """ + if self._result is None or self.points is None: + raise ValueError("no result yet or points are None, try call match()") + origin = self.points.original_data + if isinstance(origin, xr.Dataset): + return self.to_dataset() + if isinstance(origin, pd.DataFrame): + return self.to_dataframe() + raise ValueError( + "Cannot reinject: these points come from arrays, there is no Dataset or DataFrame " + "to add the values to. Use result." + ) diff --git a/tests/test_floatmatcher.py b/tests/test_floatmatcher.py index 51c59eb..219bd52 100644 --- a/tests/test_floatmatcher.py +++ b/tests/test_floatmatcher.py @@ -3,7 +3,9 @@ import numpy as np import numpy.testing as npt +import pandas as pd import pytest +import xarray as xr from floatmatcher.methods import NearestNeighbor from floatmatcher.floatmatcher import FloatMatcher @@ -11,6 +13,20 @@ from floatmatcher.product import ERA5Product, LUTProduct +# times of the two points used with the grid_3d_ds file (t0 + 2 h, t1 - 1 h) +TIMES = np.array(["2015-01-01T02", "2015-01-01T23"], dtype="datetime64[ns]") + + +def _matched(grid_3d_ds, grid_file, *data, **names): + """FloatMatcher with the given points, matched on the grid_3d_ds file.""" + fm = FloatMatcher() + fm.set_points(*data, **names) + fm.set_product("era5", "local", grid_file(grid_3d_ds), ["sst"]) + fm.set_method("nearest") + fm.match() + return fm + + def test_match_nearest_get_good_node(era5_files): points = PointSet.from_arrays( lon=[11.1, 9.05, 349.9], @@ -84,9 +100,9 @@ def test_match_nearest_over_two_variable(era5_files): # ---------- builder : set_points / set_product / set_method ---------- -def test_set_points_from_arrays(): +def test_set_points(): fm = FloatMatcher() - fm.set_points_from_arrays([11.1, 9.05], [30.05, 39.95], + fm.set_points([11.1, 9.05], [30.05, 39.95], np.array(["2015-06-01T02", "2015-06-02T00"], dtype="datetime64[ns]")) assert isinstance(fm.points, PointSet) @@ -135,11 +151,81 @@ def test_set_method_unknown_param_raises(): FloatMatcher().set_method("nearest", dummy_param=50) +# ---------- builder : set_points input types ---------- + +def test_set_points_without_time(): + fm = FloatMatcher() + fm.set_points([11.1, 9.05], [30.05, 39.95]) + + assert fm.points.time is None + + +def test_set_points_from_dataset(): + ds = xr.Dataset({"LONGITUDE": ("N_PROF", [20.1, 10.05]), + "LATITUDE": ("N_PROF", [1.05, 1.95]), + "TIME": ("N_PROF", TIMES)}) + fm = FloatMatcher() + fm.set_points(ds) + + npt.assert_allclose(fm.points.lon, [20.1, 10.05]) + assert fm.points.original_data is ds + assert fm.points.points_dim == "N_PROF" + + +def test_set_points_from_dataset_with_names(): + ds = xr.Dataset({"x": ("obs", [20.1, 10.05]), "y": ("obs", [1.05, 1.95]), "t": ("obs", TIMES)}) + fm = FloatMatcher() + fm.set_points(ds, lon="x", lat="y", time="t") + + npt.assert_allclose(fm.points.lat, [1.05, 1.95]) + assert fm.points.points_dim == "obs" + + +def test_set_points_from_dataframe(): + df = pd.DataFrame({"longitude": [20.1, 10.05], "latitude": [1.05, 1.95], "date": TIMES}) + fm = FloatMatcher() + fm.set_points(df) + + npt.assert_allclose(fm.points.lon, [20.1, 10.05]) + assert fm.points.original_data is df + + +def test_set_points_from_dataframe_with_names(): + df = pd.DataFrame({"LON": [20.1, 10.05], "LAT": [1.05, 1.95], "DATE": TIMES}) + fm = FloatMatcher() + fm.set_points(df, lon="LON", lat="LAT", time="DATE") + + npt.assert_allclose(fm.points.lat, [1.05, 1.95]) + assert len(fm.points.time) == 2 + + +def test_set_points_from_a_pointset(): + points = PointSet.from_arrays([20.1], [1.05]) + fm = FloatMatcher() + fm.set_points(points) + + assert fm.points is points + + +def test_set_points_unsupported_input_raises(): + with pytest.raises(TypeError): + FloatMatcher().set_points("not points") + with pytest.raises(TypeError): + FloatMatcher().set_points() + + +def test_set_points_names_only_for_dataframe_or_dataset(): + with pytest.raises(TypeError): + FloatMatcher().set_points([20.1], [1.05], lon="x") + with pytest.raises(TypeError): + FloatMatcher().set_points(PointSet.from_arrays([20.1], [1.05]), lon="x") + + # ---------- builder : match through the set_* ---------- def test_match_through_the_builder(era5_files): fm = FloatMatcher() - fm.set_points_from_arrays([11.1], [30.05], np.array(["2015-06-01T02"], dtype="datetime64[ns]")) + fm.set_points([11.1], [30.05], np.array(["2015-06-01T02"], dtype="datetime64[ns]")) fm.set_product("era5", "local", era5_files, ["sst"]) fm.set_method("nearest") @@ -150,6 +236,111 @@ def test_match_through_the_builder(era5_files): npt.assert_allclose(res.values["sst"], [121.03], atol=1e-4) +# ---------- builder : reinject through the set_points input ---------- +# grid_3d_ds, sst = 100*lat + lon + time_index: p0 -> node (1, 20) at t0 = 120, p1 -> node (2, 10) at t1 = 211 + +def test_to_dataset_through_the_builder(grid_3d_ds, grid_file): + ds = xr.Dataset({"LONGITUDE": ("N_PROF", [20.1, 10.05]), + "LATITUDE": ("N_PROF", [1.05, 1.95]), + "TIME": ("N_PROF", TIMES)}, + coords={"N_PROF": [10, 11]}) + fm = FloatMatcher() + fm.set_points(ds) + fm.set_product("era5", "local", grid_file(grid_3d_ds), ["sst"]) + fm.set_method("nearest") + fm.match() + + out = fm.to_dataset() + + assert out["sst_coloc"].dims == ("N_PROF",) + npt.assert_allclose(out["sst_coloc"].values, [120.0, 211.0]) + assert "sst_coloc" not in ds.data_vars # the source is not modified + + +def test_to_dataframe_through_the_builder(grid_3d_ds, grid_file): + df = pd.DataFrame({"longitude": [20.1, 10.05], "latitude": [1.05, 1.95], "date": TIMES}) + fm = FloatMatcher() + fm.set_points(df) + fm.set_product("era5", "local", grid_file(grid_3d_ds), ["sst"]) + fm.set_method("nearest") + fm.match() + + out = fm.to_dataframe() + + npt.assert_allclose(out["sst_coloc"].values, [120.0, 211.0]) + assert "sst_coloc" not in df.columns # the source is not modified + + +def test_reinject_before_match_raises(): + fm = FloatMatcher() + fm.set_points([20.1], [1.05]) + + with pytest.raises(ValueError, match="no result yet"): + fm.to_dataset() + with pytest.raises(ValueError, match="no result yet"): + fm.to_dataframe() + + +def test_to_dataframe_refuses_points_from_a_dataset(grid_3d_ds, grid_file): + ds = xr.Dataset({"LONGITUDE": ("N_PROF", [20.1, 10.05]), + "LATITUDE": ("N_PROF", [1.05, 1.95]), + "TIME": ("N_PROF", TIMES)}) + fm = _matched(grid_3d_ds, grid_file, ds) + + with pytest.raises(TypeError, match="to_dataset"): + fm.to_dataframe() + + +def test_to_dataset_refuses_points_from_a_dataframe(grid_3d_ds, grid_file): + df = pd.DataFrame({"longitude": [20.1, 10.05], "latitude": [1.05, 1.95], "date": TIMES}) + fm = _matched(grid_3d_ds, grid_file, df) + + with pytest.raises(TypeError, match="to_dataframe"): + fm.to_dataset() + + +def test_reinject_refuses_points_from_arrays(grid_3d_ds, grid_file): + """arrays carry no origin to reinject into""" + fm = _matched(grid_3d_ds, grid_file, [20.1, 10.05], [1.05, 1.95], TIMES) + + with pytest.raises(ValueError): + fm.to_dataset() + with pytest.raises(ValueError): + fm.to_dataframe() + + +def test_reinject_gives_a_dataset_for_points_from_a_dataset(grid_3d_ds, grid_file): + ds = xr.Dataset({"LONGITUDE": ("N_PROF", [20.1, 10.05]), + "LATITUDE": ("N_PROF", [1.05, 1.95]), + "TIME": ("N_PROF", TIMES)}) + fm = _matched(grid_3d_ds, grid_file, ds) + + out = fm.reinject() + + assert isinstance(out, xr.Dataset) + npt.assert_allclose(out["sst_coloc"].values, [120.0, 211.0]) + + +def test_reinject_gives_a_dataframe_for_points_from_a_dataframe(grid_3d_ds, grid_file): + df = pd.DataFrame({"longitude": [20.1, 10.05], "latitude": [1.05, 1.95], "date": TIMES}) + fm = _matched(grid_3d_ds, grid_file, df) + + out = fm.reinject() + + assert isinstance(out, pd.DataFrame) + npt.assert_allclose(out["sst_coloc"].values, [120.0, 211.0]) + + +# ---------- repr ---------- + +def test_repr_before_and_after_match(grid_3d_ds, grid_file): + fm = FloatMatcher() + assert "results: None" in repr(fm) # printing an empty FloatMatcher works + + fm = _matched(grid_3d_ds, grid_file, [20.1], [1.05], TIMES[:1]) + assert "nb_match_found" in repr(fm) + + # ---------- match : missing inputs ---------- def _points(): From 343cea62b15eb4682e3c675995f4f2639e5567ab Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Fri, 9 Oct 2026 12:35:39 +0000 Subject: [PATCH 03/10] remove variables from floatmatcher inputs (in methods now) + docstring --- src/floatmatcher/floatmatcher.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/floatmatcher/floatmatcher.py b/src/floatmatcher/floatmatcher.py index 4d0c8c3..8c8be2a 100644 --- a/src/floatmatcher/floatmatcher.py +++ b/src/floatmatcher/floatmatcher.py @@ -17,6 +17,13 @@ class FloatMatcher: Fill it with the ``set_*`` methods (points, product, method), then call ``match()`` and read the result with ``result`` or ``reinject()``. + Parameters + ---------- + points : PointSet, optional + product : Product, optional + method : Method, optional + Objects already built. Prefer the ``set_*`` methods. + Attributes ---------- points : PointSet or None @@ -41,13 +48,11 @@ class FloatMatcher: def __init__(self, points: PointSet | None = None, product: Product | None = None, - variables: str | list[str] | None = None, method: Method | None = None ) -> None: self.points = points self.product = product - self.variables = variables self.method = method self._result: MatchupResult | None = None @@ -55,7 +60,7 @@ def __init__(self, def __repr__(self,) -> str: return (f"points:{self.points} \nproduct:{self.product} " - f"\nvariables:{self.variables} \nmethod: {self.method}" + f"\nmethod: {self.method}" f"\nresults: {self._result}") From 795e2546082abda790295e56aa1832c28be53d0f Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Fri, 9 Oct 2026 13:33:30 +0000 Subject: [PATCH 04/10] docstring PointSet.py --- src/floatmatcher/pointset.py | 141 +++++++++++++++++++++++++++++------ tests/test_pointset.py | 36 ++++++++- 2 files changed, 153 insertions(+), 24 deletions(-) diff --git a/src/floatmatcher/pointset.py b/src/floatmatcher/pointset.py index 34c91e2..2d4c3db 100644 --- a/src/floatmatcher/pointset.py +++ b/src/floatmatcher/pointset.py @@ -1,13 +1,14 @@ # pointset.py: validated container for the input points to colocalize +import warnings from dataclasses import dataclass, field import numpy as np import xarray as xr import pandas as pd -from .exceptions import ProfileFormatError from numpy.typing import ArrayLike, NDArray +from .exceptions import ProfileFormatError from .geo import lonlat_to_xyz from .constants import TIME_UNIT from .utils import extract, get @@ -15,19 +16,39 @@ @dataclass(repr=False) class PointSet: - """Validated wrapper around the (lon, lat, time) arrays to colocalize. - - The wrapper carries validation and clear names, but heavy computation - works directly on the underlying NumPy arrays (``points.lon``), never on - the object itself inside loops. - - Dataset travels along original_data + """Validated positions (lon, lat, time) to match + + Inputs are converted to NumPy arrays (float degrees, ``datetime64[ns]``) and checked for + equal lengths. Heavy computation works on these arrays (``points.lon``), never on the + object itself inside loops. Prefer :meth:`from_arrays`, :meth:`from_dataframe` or + :meth:`from_xrdataset`. + + Parameters + ---------- + lon, lat: array-like of float + Degrees. + time: array-like of datetime64, optional + Time of each point. + points_dim: str, optional + Dimension holding the points in ``original_data``. + original_data: pandas.DataFrame or xarray.Dataset, optional + Object the points come from, kept to reinject the matched values. + + Raises + ------ + ValueError + If lon, lat and time do not have the same length. + + Examples + -------- + .. code-block:: python + + points = PointSet.from_arrays(lon, lat, time) """ lon: NDArray[np.float64] lat: NDArray[np.float64] time: NDArray[np.datetime64] | None = None - # TODO: introduce PointSetOrigin class (ds and df) points_dim: str | None = None # source dimension name original_data: pd.DataFrame | xr.Dataset | None = None # source dataset _xyz: NDArray[np.float64] | None = field(default=None, init=False, repr=False) @@ -41,6 +62,7 @@ def __post_init__(self) -> None: self.check_lengths() def check_lengths(self) -> None: + """Raise ValueError if lon, lat and time (when set) differ in length""" if self.time is not None: lengths = [len(self.lon), len(self.lat), len(self.time)] else: @@ -60,7 +82,7 @@ def __repr__(self) -> str: @property def xyz(self) -> NDArray[np.float64]: - """Cartesian 3D coordinates on the sphere, computed once and cached.""" + """Cartesian coordinates (km) on the sphere, shape (n, 3), computed once and cached""" if self._xyz is None: self._xyz = lonlat_to_xyz(self.lon, self.lat) return self._xyz @@ -71,6 +93,18 @@ def from_arrays(cls, lon: ArrayLike, lat: ArrayLike, time: ArrayLike | None = None) -> "PointSet": + """Build from lon, lat[, time] arrays + + Parameters + ---------- + lon, lat: array-like of float + Degrees. + time: array-like of datetime64, optional + + Returns + ------- + :class:`PointSet` + """ # Nothing to extract, no origin_* (bare arrays). Converted in __post_init__. return cls(lon, lat, time) # type: ignore[arg-type] @@ -79,12 +113,50 @@ def from_arrays(cls, @classmethod def from_dataframe(cls, df: pd.DataFrame | pd.Series, lon: str = "longitude", lat: str = "latitude", time: str | None = "date") -> "PointSet": + """Build from the columns of a DataFrame, or from one row (Series) + + The DataFrame is kept in ``original_data`` (a row becomes a one-row DataFrame) and + the name of its index in ``points_dim``. + + Parameters + ---------- + df: pandas.DataFrame or pandas.Series + lon: str, default="longitude" + lat: str, default="latitude" + time: str or None, default="date" + Time column. None gives points without time, a column that does not exist too + but with a warning. + + Returns + ------- + :class:`PointSet` + + Raises + ------ + ProfileFormatError + If the lon or lat column does not exist. + + Warns + ----- + UserWarning + If the time column does not exist. + """ + # Check the names first: a clear error for lon/lat, a warning for the optional time + missing = [name for name in (lon, lat) if name not in df] + if missing: + raise ProfileFormatError( + f"No column named {missing} in the DataFrame, available: {list(df.keys())}") + if time is not None and time not in df: + warnings.warn(f"No column named {time!r}: the points have no time. " + "Pass time=None to silence this warning.", stacklevel=2) + time = None + # Stable, named columns -> NO multi-name lookup here. # atleast_1d: a single row (pd.Series) gives scalars, turned into 1-element arrays lon_arr = np.atleast_1d(np.asarray(df[lon])) lat_arr = np.atleast_1d(np.asarray(df[lat])) time_arr = None - if time is not None and time in df: + if time is not None: time_arr = np.atleast_1d(np.asarray(df[time])) points_dim = str(df.index.name) if df.index.name is not None else "index" @@ -94,7 +166,35 @@ def from_dataframe(cls, df: pd.DataFrame | pd.Series, lon: str = "longitude", # --- 3. xarray Dataset --- @classmethod def from_xrdataset(cls, ds: xr.Dataset, lon: str = "LONGITUDE", - lat: str = "LATITUDE", time: str = "TIME") -> "PointSet": + lat: str = "LATITUDE", time: str | None = "TIME") -> "PointSet": + """Build from the variables of a Dataset (Argo names by default) + + The Dataset is kept in ``original_data`` and the dimension of the longitude variable + in ``points_dim``. + + Parameters + ---------- + ds: xarray.Dataset + lon: str, default="LONGITUDE" + lat: str, default="LATITUDE" + time: str or None, default="TIME" + Time variable, ``"JULD"`` when it is not found. None gives points without time, + no such variable too but with a warning. + + Returns + ------- + :class:`PointSet` + + Raises + ------ + ProfileFormatError + If the lon or lat variable is missing. + + Warns + ----- + UserWarning + If neither the time variable nor ``"JULD"`` exists. + """ # The point-dimension name is DISCOVERED (da_lon.dims[0]), never assumed. da_lon = get(ds, lon) point_dim = da_lon.dims[0] # DISCOVERED, never assumed @@ -103,15 +203,14 @@ def from_xrdataset(cls, ds: xr.Dataset, lon: str = "LONGITUDE", lon_arr = da_lon.values lat_arr = extract(ds, lat) time_arr = None - candidates = [time, "JULD"] - for name in candidates: - if name in ds.coords or name in ds.variables: - time_arr = extract(ds, name) - break - if time_arr is None: - raise ProfileFormatError( - f"No dataset name matches time candidates: {candidates}") - + if time is not None: + for name in [time, "JULD"]: + if name in ds.coords or name in ds.variables: + time_arr = extract(ds, name) + break + if time_arr is None: + warnings.warn(f"No variable named {time!r} or 'JULD': the points have no time. " + "Pass time=None to silence this warning.", stacklevel=2) return PointSet(lon=lon_arr, lat=lat_arr, diff --git a/tests/test_pointset.py b/tests/test_pointset.py index 66016b4..1080090 100644 --- a/tests/test_pointset.py +++ b/tests/test_pointset.py @@ -1,4 +1,6 @@ # tests/test_pointset.py +import warnings + import numpy as np import numpy.testing as npt import pandas as pd @@ -201,13 +203,23 @@ def test_from_dataframe_carries_the_original_dataframe(df_datetime_index): assert ps.points_dim == "profile_date" -def test_from_dataframe_without_time_column(df_datetime_index): - ps = PointSet.from_dataframe(df_datetime_index.drop(columns="date")) +def test_from_dataframe_without_time_column_warns(df_datetime_index): + with pytest.warns(UserWarning, match="no time"): + ps = PointSet.from_dataframe(df_datetime_index.drop(columns="date")) assert ps.time is None +def test_from_dataframe_missing_lon_or_lat_raises(df_datetime_index): + with pytest.raises(ProfileFormatError, match="latitude"): + PointSet.from_dataframe(df_datetime_index.drop(columns="latitude")) + with pytest.raises(ProfileFormatError, match="available"): + PointSet.from_dataframe(df_datetime_index, lon="lon") # wrong name given + + def test_from_dataframe_time_none(df_datetime_index): - ps = PointSet.from_dataframe(df_datetime_index, time=None) + with warnings.catch_warnings(): + warnings.simplefilter("error") # time=None is explicit: no warning + ps = PointSet.from_dataframe(df_datetime_index.drop(columns="date"), time=None) assert ps.time is None @@ -232,6 +244,24 @@ def test_from_xrdataset_time_juld_tolerance(juld_ds): dtype="datetime64[ns]")) +def test_from_xrdataset_without_time_variable(): + ds = xr.Dataset({ + "LONGITUDE": (("obs",), np.array([-45.0, -44.0])), + "LATITUDE": (("obs",), np.array([32.0, 33.0])), + }) + with pytest.warns(UserWarning, match="no time"): + ps = PointSet.from_xrdataset(ds) + assert ps.time is None + assert ps.points_dim == "obs" + + +def test_from_xrdataset_time_none(argopy_like_ds): + with warnings.catch_warnings(): + warnings.simplefilter("error") # time=None is explicit: no warning + ps = PointSet.from_xrdataset(argopy_like_ds, time=None) + assert ps.time is None + + def test_from_xrdataset_user_can_override_names(): ds = xr.Dataset({ "my_lon": (("obs",), np.array([-45.0, -44.0])), From 0452b0571247f1e1895b16b2d5fa1dcda8236e50 Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Fri, 9 Oct 2026 13:47:25 +0000 Subject: [PATCH 05/10] docstring products --- src/floatmatcher/product.py | 103 ++++++++++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 4 deletions(-) diff --git a/src/floatmatcher/product.py b/src/floatmatcher/product.py index da92420..8f9ce41 100644 --- a/src/floatmatcher/product.py +++ b/src/floatmatcher/product.py @@ -18,9 +18,20 @@ # ───────────────────────────────────────────────────────────── def to_standard(ds: xr.Dataset, mapping: dict[str, str]) -> xr.Dataset: - """ - Rename keys found in mapping. If key not found, skip and let it as it is. - set variables lon/lat/time as coords if not already + """Rename the dataset names found in a mapping to standard names + + Names of the mapping that are not in the dataset are skipped. The ``lon``, ``lat`` and + ``time`` data variables are set as coordinates. + + Parameters + ---------- + ds: xarray.Dataset + mapping: dict + Names of the product -> standard names. + + Returns + ------- + :class:`xarray.Dataset` """ renamable = set(ds.variables) | set(ds.dims) @@ -43,6 +54,42 @@ def to_standard(ds: xr.Dataset, mapping: dict[str, str]) -> xr.Dataset: class Product(ABC): + """Gridded product the points are matched with + + A product knows where its files come from (its source), opens them as one dataset with + standard names (lon, lat, time) and keeps it in ``src_dataset``. Subclasses give their + ``name``, ``coord_map`` and ``src_available``, and implement ``normalize``. + + Parameters + ---------- + source: str + Where the files come from, one of ``src_available``. + path: str or list of str + Directory or files of the product. + selected_variables: list of str or None + Variables to keep. None keeps all. + **source_params + Parameters of the source. ``"local"``: ``pattern``. + + Attributes + ---------- + name: str + Key of the product in ``available_products``. + coord_map: dict + Names of the product -> standard names. + src_available: list of str + Sources the product can be read from. + src_dataset: xarray.Dataset or None + Complete dataset, set by ``open()``. + + Raises + ------ + ValueError + If ``source`` is not in ``src_available``, or ``selected_variables`` is neither None + nor a non-empty list. + TypeError + If a source parameter is unknown. + """ name: str coord_map: dict[str, str] = {} src_available: list[str] = [] @@ -76,12 +123,20 @@ def __init__(self, # avoid duplication of path storage into Product and Source @property def path(self) -> str | Path | list[str]: + """Directory or files of the product, as given to the source""" return self._source.path # view of src_dataset restricted to selected_variables : no data copy, # src_dataset stays complete @property def selected_dataset(self) -> xr.Dataset: + """Dataset restricted to ``selected_variables``, without copying the data + + Raises + ------ + ValueError + If the product is not opened. + """ if self.src_dataset is None: raise ValueError("product not opened, call open(points) first") return _select_variables(self.src_dataset, self.selected_variables) @@ -89,6 +144,13 @@ def selected_dataset(self) -> xr.Dataset: # property of available variables in dataset @property def available_variables(self) -> list[str]: + """Names of the variables of the opened dataset + + Raises + ------ + ValueError + If the product is not opened. + """ if self.src_dataset is None: raise ValueError("product not opened, call open(points) first") return [str(v) for v in self.src_dataset.data_vars] @@ -139,7 +201,21 @@ def _check_time_unicity(self, dataset: xr.Dataset) -> None: raise ValueError("grid: duplicate timestamps (overlapping files?)") def open(self, points: PointSet) -> None: - """ open files and src_dataset """ + """Open the files needed by the points as one dataset + + The dataset is normalized, checked (lon and lat, at least one variable, no duplicated + longitude, latitude or timestamp) and kept in ``src_dataset``. + + Parameters + ---------- + points: :class:`PointSet` + Positions to match, they decide which files are needed. + + Raises + ------ + ValueError + If there is no file, or if the dataset fails a check. + """ files = self._source.resolve(points) ds = self._open_paths(files) ds = self.normalize(ds) @@ -153,10 +229,25 @@ def __repr__(self) -> str: @abstractmethod def normalize(self, ds_raw: xr.Dataset) -> xr.Dataset: + """Rename the names of the raw dataset to standard names (lon, lat, time) + + Parameters + ---------- + ds_raw: xarray.Dataset + Dataset as opened from the files. + + Returns + ------- + :class:`xarray.Dataset` + """ ... class ERA5Product(Product): + """ERA5 reanalysis of ECMWF + + The names ``longitude``, ``latitude`` and ``valid_time`` are renamed lon, lat and time. + """ name = "era5" coord_map = {"longitude": "lon", "latitude": "lat", "valid_time": "time"} src_available = ["local"] @@ -166,6 +257,10 @@ def normalize(self, ds_raw: xr.Dataset) -> xr.Dataset: return ds class LUTProduct(Product): + """Look-up table (LUT) gridded product + + Its coordinates are already named lon and lat. + """ name = "lut" coord_map = {"lon": "lon", "lat": "lat"} src_available = ["local"] From 798d6e134dd309872230bc722a39b096d61cc533 Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Mon, 12 Oct 2026 09:07:34 +0000 Subject: [PATCH 06/10] neighbors into flatgrid --- src/floatmatcher/flatgrid.py | 67 +++++++++++++++++++++++++++++++++- src/floatmatcher/methods.py | 10 +---- src/floatmatcher/neighbors.py | 44 ---------------------- tests/test_flatgrid.py | 69 +++++++++++++++++++++++++++++++++++ tests/test_neighbors.py | 63 -------------------------------- 5 files changed, 137 insertions(+), 116 deletions(-) delete mode 100644 src/floatmatcher/neighbors.py delete mode 100644 tests/test_neighbors.py diff --git a/src/floatmatcher/flatgrid.py b/src/floatmatcher/flatgrid.py index 3a7a8f6..502626b 100644 --- a/src/floatmatcher/flatgrid.py +++ b/src/floatmatcher/flatgrid.py @@ -1,12 +1,16 @@ -# flatgrid.py: the reference point cloud (flattened grid), internal to nearest +# flatgrid.py: the reference point cloud (flattened grid), internal to nearest, +# with the spatial and temporal nearest lookups (KDTree) on it from dataclasses import dataclass, field import numpy as np import xarray as xr from numpy.typing import ArrayLike, NDArray +from scipy.spatial import cKDTree +from .constants import REF_TIME, TIME_UNIT from .geo import lonlat_to_xyz +from .pointset import PointSet @dataclass @@ -47,3 +51,64 @@ def read_values(self, node_idx: ArrayLike, sel = da_var.isel(node=node, time=xr.DataArray(np.asarray(tsel_idx), dims="pts")) out[str(var)] = sel.values return out + + def spatial_nearest(self, points: PointSet, + k: int = 1) -> tuple[NDArray[np.float64], NDArray[np.int64]]: + """Nearest nodes of the points in space + + Parameters + ---------- + points: :class:`PointSet` + k: int, default=1 + Number of neighbours. + + Returns + ------- + tuple of arrays + Distances (km) and indices of the nodes. + """ + dist: NDArray[np.float64] + idx: NDArray[np.int64] + dist, idx = cKDTree(self.xyz).query(points.xyz, k=k) + return dist, idx + + def temporal_nearest(self, points: PointSet, + k: int = 1) -> tuple[NDArray[np.float64], NDArray[np.int64]]: + """Nearest time steps of the points + + Parameters + ---------- + points: :class:`PointSet` + k: int, default=1 + Number of neighbours. + + Returns + ------- + tuple of arrays + Time gaps (seconds) and indices of the time steps. + + Raises + ------ + ValueError + If the grid or the points have no time. + """ + time_delta: NDArray[np.float64] + idx: NDArray[np.int64] + if self.time is None: + raise ValueError("the grid has no time axis") + if points.time is None: + raise ValueError("points have no time: they cannot be matched on a grid with a time axis") + grid_tree = cKDTree(_to_seconds(self.time)[:, None]) + time_delta, idx = grid_tree.query(_to_seconds(points.time)[:, None], k=k) + return time_delta, idx + + +def _to_seconds(times: NDArray[np.datetime64]) -> NDArray[np.float64]: + """Convert datetime64 to floating-point seconds since a fixed epoch. + + Working in a common float unit lets the 1D KDTree measure time distance, + and returning *seconds* makes the time_gap_seconds constraint directly comparable. + """ + delta = np.asarray(times, dtype=f"datetime64[{TIME_UNIT}]") - REF_TIME + seconds: NDArray[np.float64] = delta / np.timedelta64(1, "s") + return seconds diff --git a/src/floatmatcher/methods.py b/src/floatmatcher/methods.py index eb03fc7..3e88cc6 100644 --- a/src/floatmatcher/methods.py +++ b/src/floatmatcher/methods.py @@ -12,7 +12,6 @@ from .flatgrid import FlatGrid from .matchup_results import MatchupResult -from .neighbors import spatial_nearest, temporal_nearest from .pointset import PointSet from .utils import indented_repr @@ -48,19 +47,14 @@ def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: # starting by lonlat2xy on spatial grid # return FlatGrid object, flatten grid flat_grid = FlatGrid.from_grid(dataset) - # convert into carthesian coordinates - grid_stacked = flat_grid.xyz # starting Nearest method : apply kdtree on spatial - dist_km, spatial_idx = spatial_nearest(grid_stacked, points, k=self.k_nearest) + dist_km, spatial_idx = flat_grid.spatial_nearest(points, k=self.k_nearest) valid_spatial = dist_km <= self.radius idx_count = len(points.lon) if flat_grid.time is not None: - time_delta, temporal_idx = temporal_nearest(flat_grid.time, - points, - k=self.k_nearest - ) + time_delta, temporal_idx = flat_grid.temporal_nearest(points, k=self.k_nearest) valid = valid_spatial & (time_delta <= self.time_gap_seconds) else: time_delta = np.full(idx_count, np.nan) diff --git a/src/floatmatcher/neighbors.py b/src/floatmatcher/neighbors.py deleted file mode 100644 index 35992d6..0000000 --- a/src/floatmatcher/neighbors.py +++ /dev/null @@ -1,44 +0,0 @@ -# neighbors.py: spatial and temporal KDTree lookups over a reference grid. -# -# Spatial and temporal are SEPARATED because they now have different lifetimes: -# - SpatialIndex is built ONCE (grid geometry is identical on every packet); -# - TemporalIndex is built PER packet (only the time axis changes). -# On a regular grid the spatial positions repeat at every time step, so -# "closest in space" and "closest in time" are independent questions. - -import numpy as np -from numpy.typing import NDArray -from scipy.spatial import cKDTree - -from .pointset import PointSet -from .constants import REF_TIME, TIME_UNIT - - -def _to_seconds(times: NDArray[np.datetime64]) -> NDArray[np.float64]: - """Convert datetime64 to floating-point seconds since a fixed epoch. - - Working in a common float unit lets the 1D KDTree measure time distance, - and returning *seconds* makes the time_gap_seconds constraint directly comparable. - """ - delta = np.asarray(times, dtype=f"datetime64[{TIME_UNIT}]") - REF_TIME - seconds: NDArray[np.float64] = delta / np.timedelta64(1, "s") - return seconds - - -def spatial_nearest(grid_xyz: NDArray[np.float64], points: PointSet, - k: int = 1) -> tuple[NDArray[np.float64], NDArray[np.int64]]: - dist: NDArray[np.float64] - idx: NDArray[np.int64] - dist, idx = cKDTree(grid_xyz).query(points.xyz, k=k) - return dist, idx - -def temporal_nearest(grid_times: NDArray[np.datetime64], points: PointSet, - k: int = 1) -> tuple[NDArray[np.float64], NDArray[np.int64]]: - time_delta: NDArray[np.float64] - idx: NDArray[np.int64] - if points.time is None: - raise ValueError("points have no time: they cannot be matched on a grid with a time axis") - grid_tree = cKDTree(_to_seconds(grid_times)[:, None]) - time_delta, idx = grid_tree.query(_to_seconds(points.time)[:, None], k=k) - return time_delta, idx - diff --git a/tests/test_flatgrid.py b/tests/test_flatgrid.py index 348bf87..664edf7 100644 --- a/tests/test_flatgrid.py +++ b/tests/test_flatgrid.py @@ -5,8 +5,11 @@ import numpy as np import numpy.testing as npt +import pytest from floatmatcher.flatgrid import FlatGrid +from floatmatcher.pointset import PointSet +from helpers import make_grid # Reminder # 3D (grid_3d_ds): @@ -70,3 +73,69 @@ def test_xyz_is_cached(grid_2d_ds): test if @property works well""" flat_grid = FlatGrid.from_grid(grid_2d_ds) assert flat_grid.xyz is flat_grid.xyz + + + +# ---------- nearest lookups ---------- +# 2x2 grid, nodes (lat, lon): 0 = (0, 0), 1 = (0, 10), 2 = (10, 0), 3 = (10, 10) + +GRID_TIMES = np.array(["2015-01-01", "2015-01-02", "2015-01-03", + "2015-01-04", "2015-01-05"], dtype="datetime64[ns]") + + +@pytest.fixture +def flat_2d(): + return FlatGrid.from_grid(make_grid(lat=[0.0, 10.0], lon=[0.0, 10.0])) + + +@pytest.fixture +def flat_3d(): + return FlatGrid.from_grid(make_grid(lat=[0.0, 10.0], lon=[0.0, 10.0], time=GRID_TIMES)) + + +@pytest.fixture +def points_on_nodes(): + """Three points sitting exactly on grid nodes 2, 0 and 3.""" + return PointSet( + lon=[0.0, 0.0, 10.0], + lat=[10.0, 0.0, 10.0], + time=np.array(["2015-01-03T03:00", "2015-01-01T00:00", "2015-01-04T23:00"], + dtype="datetime64[ns]"), + ) + + +def test_spatial_nearest_finds_the_node_under_each_point(flat_2d, points_on_nodes): + dist, idx = flat_2d.spatial_nearest(points_on_nodes) + npt.assert_array_equal(idx, [2, 0, 3]) + npt.assert_allclose(dist, 0.0, atol=1e-9) + + +def test_spatial_nearest_distance_is_in_kilometres(flat_2d): + points = PointSet(lon=[1.0], lat=[0.0], + time=np.array(["2015-01-01"], dtype="datetime64[ns]")) + dist, idx = flat_2d.spatial_nearest(points) + expected = np.linalg.norm(points.xyz[0] - flat_2d.xyz[0]) + npt.assert_array_equal(idx, [0]) + npt.assert_allclose(dist, [expected]) + + +def test_spatial_nearest_k_returns_sorted_neighbours(flat_2d, points_on_nodes): + dist, idx = flat_2d.spatial_nearest(points_on_nodes, k=3) + assert dist.shape == (3, 3) + assert idx.shape == (3, 3) + npt.assert_array_equal(idx[:, 0], [2, 0, 3]) + assert (np.diff(dist, axis=1) >= 0).all() + + +def test_temporal_nearest_delta_is_in_seconds(flat_3d, points_on_nodes): + time_delta, idx = flat_3d.temporal_nearest(points_on_nodes) + npt.assert_array_equal(idx, [2, 0, 4]) + npt.assert_allclose(time_delta, [10800.0, 0.0, 3600.0]) + + +def test_temporal_nearest_k_returns_sorted_neighbours(flat_3d, points_on_nodes): + time_delta, idx = flat_3d.temporal_nearest(points_on_nodes, k=2) + assert idx.shape == (3, 2) + npt.assert_array_equal(idx[:, 0], [2, 0, 4]) + npt.assert_allclose(time_delta[:, 0], [10800.0, 0.0, 3600.0]) + assert (np.diff(time_delta, axis=1) >= 0).all() diff --git a/tests/test_neighbors.py b/tests/test_neighbors.py deleted file mode 100644 index b48a4d7..0000000 --- a/tests/test_neighbors.py +++ /dev/null @@ -1,63 +0,0 @@ -# tests/test_neighbors.py - -import numpy as np -import numpy.testing as npt -import pytest - -from floatmatcher.geo import lonlat_to_xyz -from floatmatcher.neighbors import spatial_nearest, temporal_nearest -from floatmatcher.pointset import PointSet - -grid_lon = [0.0, 10.0, 0.0, 10.0] -grid_lat = [0.0, 0.0, 10.0, 10.0] -grid_xyz = lonlat_to_xyz(grid_lon, grid_lat) -grid_times = np.array(["2015-01-01", "2015-01-02", "2015-01-03", - "2015-01-04", "2015-01-05"], dtype="datetime64[ns]") - - -@pytest.fixture -def points_on_nodes(): - """Three points sitting exactly on grid nodes 2, 0 and 3.""" - return PointSet( - lon=[0.0, 0.0, 10.0], - lat=[10.0, 0.0, 10.0], - time=np.array(["2015-01-03T03:00", "2015-01-01T00:00", "2015-01-04T23:00"], - dtype="datetime64[ns]"), - ) - - -def test_spatial_nearest_finds_the_node_under_each_point(points_on_nodes): - dist, idx = spatial_nearest(grid_xyz, points_on_nodes) - npt.assert_array_equal(idx, [2, 0, 3]) - npt.assert_allclose(dist, 0.0, atol=1e-9) - - -def test_spatial_nearest_distance_is_in_kilometres(): - points = PointSet(lon=[1.0], lat=[0.0], - time=np.array(["2015-01-01"], dtype="datetime64[ns]")) - dist, idx = spatial_nearest(grid_xyz, points) - expected = np.linalg.norm(points.xyz[0] - grid_xyz[0]) - npt.assert_array_equal(idx, [0]) - npt.assert_allclose(dist, [expected]) - - -def test_spatial_nearest_k_returns_sorted_neighbours(points_on_nodes): - dist, idx = spatial_nearest(grid_xyz, points_on_nodes, k=3) - assert dist.shape == (3, 3) - assert idx.shape == (3, 3) - npt.assert_array_equal(idx[:, 0], [2, 0, 3]) - assert (np.diff(dist, axis=1) >= 0).all() - - -def test_temporal_nearest_delta_is_in_seconds(points_on_nodes): - time_delta, idx = temporal_nearest(grid_times, points_on_nodes) - npt.assert_array_equal(idx, [2, 0, 4]) - npt.assert_allclose(time_delta, [10800.0, 0.0, 3600.0]) - - -def test_temporal_nearest_k_returns_sorted_neighbours(points_on_nodes): - time_delta, idx = temporal_nearest(grid_times, points_on_nodes, k=2) - assert idx.shape == (3, 2) - npt.assert_array_equal(idx[:, 0], [2, 0, 4]) - npt.assert_allclose(time_delta[:, 0], [10800.0, 0.0, 3600.0]) - assert (np.diff(time_delta, axis=1) >= 0).all() From 75f9c14cb27f0ddc931c73a686442187d1aaabe9 Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Mon, 12 Oct 2026 12:12:22 +0000 Subject: [PATCH 07/10] doctrings --- src/floatmatcher/constants.py | 12 +-- src/floatmatcher/exceptions.py | 11 ++- src/floatmatcher/flatgrid.py | 50 +++++++++-- src/floatmatcher/floatmatcher.py | 125 +++++++++++++++------------- src/floatmatcher/geo.py | 13 ++- src/floatmatcher/matchup_results.py | 61 ++++++++++++-- src/floatmatcher/methods.py | 68 +++++++++++++-- src/floatmatcher/product.py | 7 +- src/floatmatcher/resolver.py | 109 +++++++++++++++++++----- src/floatmatcher/sources.py | 63 +++++++++++++- src/floatmatcher/utils.py | 87 +++++++++++++++++-- tests/helpers.py | 2 +- tests/test_products.py | 2 +- 13 files changed, 489 insertions(+), 121 deletions(-) diff --git a/src/floatmatcher/constants.py b/src/floatmatcher/constants.py index 42f3b2b..10a2929 100644 --- a/src/floatmatcher/constants.py +++ b/src/floatmatcher/constants.py @@ -4,13 +4,15 @@ # --- geography --- EARTH_RADIUS_KM: float = 6371.0 -"""Mean Earth radius (km), used to project lon/lat onto a sphere in geo.py.""" +"""float: Mean Earth radius (km), used to project lon/lat onto a sphere in geo.py""" # --- time --- TIME_UNIT: str = "ns" -"""Canonical datetime64 resolution enforced across the library (points and grids).""" +"""str: Canonical datetime64 resolution enforced across the library (points and grids)""" REF_TIME: np.datetime64 = np.datetime64("1970-01-01", "ns") -"""Reference epoch: datetime64 values are turned into floating-point days from -here for the 1D temporal KDTree. Any fixed epoch works; what matters is that -points and grid use the SAME one, so their difference is correct.""" +"""numpy.datetime64: Reference epoch of the time lookup + +Datetimes are turned into floating-point seconds from here for the 1D temporal KDTree. Any +fixed epoch works; what matters is that points and grid use the SAME one, so their +difference is correct""" diff --git a/src/floatmatcher/exceptions.py b/src/floatmatcher/exceptions.py index de9deba..fa3e51e 100644 --- a/src/floatmatcher/exceptions.py +++ b/src/floatmatcher/exceptions.py @@ -4,10 +4,15 @@ class FloatMatcherError(Exception): - """Base class for all FloatMatcher errors.""" + """Base class for all FloatMatcher errors + + Catch it to handle every error raised by the library. + """ class ProfileFormatError(FloatMatcherError): - """Raised when point data cannot be extracted from the given source - (missing coordinate, unrecognized structure).""" + """Raise when the points cannot be extracted from the given data + + A coordinate or variable is missing, or the structure is not recognized. + """ diff --git a/src/floatmatcher/flatgrid.py b/src/floatmatcher/flatgrid.py index 502626b..5580398 100644 --- a/src/floatmatcher/flatgrid.py +++ b/src/floatmatcher/flatgrid.py @@ -15,7 +15,19 @@ @dataclass class FlatGrid: - """Flattened grid as a point cloud, for KDTree lookup.""" + """Flattened grid as a point cloud + + The nodes (lat, lon) of a 2D or 3D grid are stacked in one dimension, so that each node is + a point to look up with a KDTree. The values stay lazy and are read only at the retained + nodes. Prefer :meth:`from_grid`. + + Parameters + ---------- + lon, lat: numpy.ndarray + Coordinates of the nodes, in degrees. + time: numpy.ndarray of datetime64 or None + Time steps of the grid. None for a grid without time axis. + """ lon: NDArray[np.float64] lat: NDArray[np.float64] @@ -25,14 +37,26 @@ class FlatGrid: @property def xyz(self) -> NDArray[np.float64]: - """Cartesian 3D coordinates of the nodes, computed once and cached.""" + """Cartesian coordinates (km) of the nodes on the sphere, shape (n, 3), computed once and cached""" if self._xyz is None: self._xyz = lonlat_to_xyz(self.lon, self.lat) return self._xyz @classmethod def from_grid(cls, ds: xr.Dataset) -> "FlatGrid": - """Flatten a grid (2D or 3D) into a node cloud. Values stay lazy.""" + """Flatten a grid, 2D or 3D, into a node cloud + + The values stay lazy. + + Parameters + ---------- + ds: xarray.Dataset + Grid with the ``lat`` and ``lon`` coordinates, and ``time`` if it has a time axis. + + Returns + ------- + :class:`FlatGrid` + """ stacked = ds.stack(node=("lat", "lon")) time = ds["time"].values if "time" in ds.coords else None return cls(lon=stacked["lon"].values, lat=stacked["lat"].values, # trigger memory import xr dataset @@ -40,7 +64,21 @@ def from_grid(cls, ds: xr.Dataset) -> "FlatGrid": def read_values(self, node_idx: ArrayLike, tsel_idx: ArrayLike | None = None) -> dict[str, NDArray[np.float64]]: - """Read variable values ONLY at the retained (node[, time]) indices""" + """Read the values of the variables only at the retained indices + + Parameters + ---------- + node_idx: array-like of int + Indices of the nodes. + tsel_idx: array-like of int, optional + Indices of the time steps, paired element by element with ``node_idx``. None for a + grid without time axis. + + Returns + ------- + dict + One array of values per variable. + """ node = xr.DataArray(np.asarray(node_idx), dims="pts") out = {} for var in self._stacked.data_vars: # _stacked is lazy ds with only lon/lat/time in memory @@ -60,7 +98,7 @@ def spatial_nearest(self, points: PointSet, ---------- points: :class:`PointSet` k: int, default=1 - Number of neighbours. + Number of neighbors. Returns ------- @@ -80,7 +118,7 @@ def temporal_nearest(self, points: PointSet, ---------- points: :class:`PointSet` k: int, default=1 - Number of neighbours. + Number of neighbors. Returns ------- diff --git a/src/floatmatcher/floatmatcher.py b/src/floatmatcher/floatmatcher.py index 8c8be2a..f848e57 100644 --- a/src/floatmatcher/floatmatcher.py +++ b/src/floatmatcher/floatmatcher.py @@ -12,37 +12,39 @@ from .product import Product, available_products class FloatMatcher: - """Match positions with a gridded product. + """Positions matcher for gridded products - Fill it with the ``set_*`` methods (points, product, method), then call ``match()`` - and read the result with ``result`` or ``reinject()``. + Fill it with the ``set_*`` methods (points, product, method), then call ``match()`` and read + the result with ``result`` or ``reinject()``. Parameters ---------- - points : PointSet, optional - product : Product, optional - method : Method, optional + points: PointSet, optional + product: Product, optional + method: Method, optional Objects already built. Prefer the ``set_*`` methods. Attributes ---------- - points : PointSet or None + points: PointSet or None Positions (lon, lat, time) to match. - product : Product or None + product: Product or None Product the points are matched with. - method : Method or None + method: Method or None Matching method and its parameters. - result : MatchupResult + result: MatchupResult Result of the last ``match()``. Raises if there is none yet. Examples -------- - >>> fm = FloatMatcher() - >>> fm.set_points(lon, lat, time) - >>> fm.set_product("era5", "local", "/data/era5", ["sst"]) - >>> fm.set_method("nearest", radius=50, time_gap=np.timedelta64(3, "h")) - >>> fm.match() - >>> fm.result + .. code-block:: python + + fm = FloatMatcher() + fm.set_points(lon, lat, time) + fm.set_product("era5", "local", "/data/era5", ["sst"]) + fm.set_method("nearest", radius=50, time_gap=np.timedelta64(3, "h")) + fm.match() + fm.result """ def __init__(self, @@ -66,21 +68,30 @@ def __repr__(self,) -> str: def set_points(self, *data: Any, **names: str) -> None: - """Set the points to match. The loader is chosen from the type of the input. + """Set the points to match + + The loader is chosen from the type of the input. Parameters ---------- *data - ``lon, lat[, time]`` arrays, a :class:`pandas.DataFrame`, a - :class:`xarray.Dataset` or a :class:`PointSet`. - **names : str - ``lon``, ``lat``, ``time``: names of the columns or variables holding them. - Only for a DataFrame or a Dataset. + ``lon, lat[, time]`` arrays, a :class:`pandas.DataFrame`, a :class:`xarray.Dataset` or a + :class:`PointSet`. + **names: str + ``lon``, ``lat``, ``time``: names of the columns or variables holding them. Only for a + DataFrame or a Dataset. Raises ------ TypeError - Unsupported input, or names given with arrays or a PointSet. + If the input is not supported, or if names are given with arrays or a PointSet. + ProfileFormatError + If the lon or lat column or variable is missing (DataFrame or Dataset). + + Warns + ----- + UserWarning + If the time column or variable is missing (DataFrame or Dataset). """ if len(data) == 1 and not names and isinstance(data[0], PointSet): self.points = data[0] @@ -105,17 +116,17 @@ def set_product(self, selected_variables: list[str] | None, **src_params: Any ) -> None: - """Set the product to match with. + """Set the product to match with Parameters ---------- - type : str - ``"era5"`` or ``"lut"``. - source : str - Where the files come from. Only ``"local"`` for now. - path : str or list of str + type: {'era5', 'lut'} + Product to use. + source: {'local'} + Where the files come from. + path: str or list of str Directory or files of the product. With a ``pattern``, the root of the tree. - selected_variables : list of str or None + selected_variables: list of str or None Variables to keep. None keeps all. **src_params Parameters of the source. ``"local"``: ``pattern``, a path template like @@ -124,9 +135,9 @@ def set_product(self, Raises ------ ValueError - Unknown product, or source not available for it. + If the product is unknown, or the source is not available for it. TypeError - Unknown source parameter. + If a source parameter is unknown. """ if type not in available_products: raise ValueError(f"unknown product {type!r}, available: {list(available_products)}") @@ -140,22 +151,22 @@ def set_product(self, def set_method(self, type: str, **params: Any) -> None: - """Set the matching method. + """Set the matching method Parameters ---------- - type : str - ``"nearest"``. + type: {'nearest'} + Method to use. **params - Parameters of the method. ``"nearest"``: ``radius`` (km), - ``time_gap`` (``numpy.timedelta64``), ``k_nearest``. + Parameters of the method. ``"nearest"``: ``radius`` (km), ``time_gap`` + (numpy.timedelta64), ``k_nearest``. Raises ------ ValueError - Unknown method. + If the method is unknown. TypeError - Unknown method parameter. + If a method parameter is unknown. """ if type not in available_methods: raise ValueError(f"unknown method {type!r}, available: {list(available_methods)}") @@ -164,12 +175,12 @@ def set_method(self, type: str, **params: Any) -> None: @property def result(self) -> MatchupResult: - """Result of the last ``match()``. + """Result of the last ``match()`` Raises ------ AttributeError - No result yet. + If there is no result yet. """ if self._result is None : raise AttributeError("no result yet, call match()") @@ -177,12 +188,14 @@ def result(self) -> MatchupResult: def match(self) -> None: - """Run the matchup. Read the result with ``result``. + """Run the matchup + + Read the result with ``result``. Raises ------ ValueError - Points, product or method not set. + If the points, the product or the method are not set. """ if self.points is None: raise ValueError("match(): no points set, call set_points(...)") @@ -199,20 +212,20 @@ def match(self) -> None: def to_dataset(self) -> xr.Dataset: - """Add the matched values to the Dataset the points come from. + """Add the matched values to the Dataset the points come from Values are added as ``_coloc``. The source Dataset is not modified. Returns ------- - xarray.Dataset + :class:`xarray.Dataset` Raises ------ ValueError - No result yet, or points not loaded from a Dataset. + If there is no result yet, or the points come from arrays. TypeError - Points loaded from a DataFrame, use ``to_dataframe()``. + If the points come from a DataFrame, use ``to_dataframe()``. """ if self._result is None or self.points is None: raise ValueError("no result yet or points are None, try call match()") @@ -220,20 +233,20 @@ def to_dataset(self) -> xr.Dataset: return self._result._to_dataset(self.points) def to_dataframe(self) -> pd.DataFrame: - """Add the matched values to the DataFrame the points come from. + """Add the matched values to the DataFrame the points come from Values are added as ``_coloc`` columns. The source DataFrame is not modified. Returns ------- - pandas.DataFrame + :class:`pandas.DataFrame` Raises ------ ValueError - No result yet, or points not loaded from a DataFrame. + If there is no result yet, or the points come from arrays. TypeError - Points loaded from a Dataset, use ``to_dataset()``. + If the points come from a Dataset, use ``to_dataset()``. """ if self._result is None or self.points is None: raise ValueError("no result yet or points are None, try call match()") @@ -241,19 +254,19 @@ def to_dataframe(self) -> pd.DataFrame: return self._result._to_dataframe(self.points) def reinject(self) -> xr.Dataset | pd.DataFrame: - """Add the matched values to the object the points come from. + """Add the matched values to the object the points come from - Same type as the input of ``set_points``: a Dataset gives a Dataset, a DataFrame - gives a DataFrame. Values are added as ``_coloc``. The source is not modified. + Same type as the input of ``set_points``: a Dataset gives a Dataset, a DataFrame gives a + DataFrame. Values are added as ``_coloc``. The source is not modified. Returns ------- - xarray.Dataset or pandas.DataFrame + :class:`xarray.Dataset` or :class:`pandas.DataFrame` Raises ------ ValueError - No result yet, or points not loaded from a Dataset or a DataFrame. + If there is no result yet, or the points come from arrays. """ if self._result is None or self.points is None: raise ValueError("no result yet or points are None, try call match()") diff --git a/src/floatmatcher/geo.py b/src/floatmatcher/geo.py index 879f585..b72f6b2 100644 --- a/src/floatmatcher/geo.py +++ b/src/floatmatcher/geo.py @@ -9,7 +9,18 @@ #_______ def lonlat_to_xyz(lon: ArrayLike, lat: ArrayLike) -> NDArray[np.float64]: - """Convert lon/lat to xyz on the shpere""" + """Convert longitudes and latitudes to Cartesian coordinates on the sphere + + Parameters + ---------- + lon, lat: array-like of float + Degrees. + + Returns + ------- + numpy.ndarray + Coordinates in km, shape (n, 3), for the radius of the Earth. + """ lon_rad = np.radians(lon) lat_rad = np.radians(lat) x = EARTH_RADIUS_KM * np.cos(lat_rad) * np.cos(lon_rad) diff --git a/src/floatmatcher/matchup_results.py b/src/floatmatcher/matchup_results.py index e6e2136..fa46374 100644 --- a/src/floatmatcher/matchup_results.py +++ b/src/floatmatcher/matchup_results.py @@ -1,4 +1,4 @@ -# results.py: output object of matchup +# matchup_results.py: output object of matchup import numpy as np import pandas as pd @@ -11,6 +11,22 @@ @dataclass class MatchupResult: + """Result of a matchup + + One value per point, in the order of the points. A point that is not matched (too far in + space or in time) has NaN values, distance and time gap. + + Parameters + ---------- + values: dict + Matched values, one array per variable. + distance_km: numpy.ndarray + Distance to the matched node, in km. + time_delta: numpy.ndarray + Gap to the matched time step, in seconds. NaN for a grid without time axis. + valid: numpy.ndarray of bool + True for the matched points. + """ values: dict[str, NDArray[np.float64]] distance_km: NDArray[np.float64] time_delta: NDArray[np.float64] @@ -24,9 +40,25 @@ def __repr__(self) -> str: def _to_dataset(self, points: PointSet) -> xr.Dataset: - """ - Reinject the colocalized values into the dataset - The source Dataset travels inside the PointSet + """Reinject the matched values into the Dataset the points come from + + Values are added as ``_coloc``, on the dimension of the points. The source is + not modified. + + Parameters + ---------- + points: :class:`PointSet` + + Returns + ------- + :class:`xarray.Dataset` + + Raises + ------ + ValueError + If the points have no origin. + TypeError + If the origin is not a Dataset. """ ds = points.original_data dim = points.points_dim @@ -48,9 +80,24 @@ def _to_dataset(self, points: PointSet) -> xr.Dataset: return out def _to_dataframe(self, points: PointSet) -> pd.DataFrame: - """ - Reinject the colocalized values into the dataframe - The source DataFrame travels inside the PointSet + """Reinject the matched values into the DataFrame the points come from + + Values are added as ``_coloc`` columns. The source is not modified. + + Parameters + ---------- + points: :class:`PointSet` + + Returns + ------- + :class:`pandas.DataFrame` + + Raises + ------ + ValueError + If the points have no origin. + TypeError + If the origin is not a DataFrame. """ df = points.original_data if df is None: diff --git a/src/floatmatcher/methods.py b/src/floatmatcher/methods.py index 3e88cc6..3ea16ce 100644 --- a/src/floatmatcher/methods.py +++ b/src/floatmatcher/methods.py @@ -1,9 +1,8 @@ -# methods.py: matchup methods. - -# The spatial half is prepared ONCE (prepare) and reused on every temporal -# packet (match_packet), because the grid geometry is identical across packets. -# The packet loop itself lives in the FloatMatcher; this module provides the -# two halves and a single-pass `match` for the non-batched case. +# methods.py: matchup methods (contract and registry). +# +# The grid is opened ONCE by the product and handed to apply(): the spatial and temporal +# lookups run on the whole grid in a single pass (no temporal packets), and the values are +# read lazily, only at the valid points. from abc import ABC, abstractmethod @@ -17,19 +16,49 @@ class Method(ABC): - """Contract shared by every matchup method.""" + """Prototype of the matchup methods + + A method holds its parameters and matches points with a grid in ``apply``. It knows + nothing of the product or of :class:`FloatMatcher`: it receives the opened grid and the + points. Subclasses implement ``apply`` and are registered in ``available_methods``. + """ def __repr__(self) -> str: return indented_repr(self) @abstractmethod def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: + """Match the points with a grid + + Parameters + ---------- + dataset: xarray.Dataset + Grid with the ``lon`` and ``lat`` coordinates, and ``time`` if it has a time axis. + points: :class:`PointSet` + + Returns + ------- + :class:`MatchupResult` + """ ... class NearestNeighbor(Method): - """Nearest-neighbor matchup method""" + """Nearest-neighbor matchup method + + Each point takes the values of the nearest node of the grid, and of the nearest time step + if the grid has a time axis. Points beyond ``radius`` or ``time_gap`` are invalid. + + Parameters + ---------- + radius: int, default=25 + Maximum distance to the node, in km. + time_gap: numpy.timedelta64, default=1 day + Maximum gap to the time step. Ignored for a grid without time axis. + k_nearest: int, default=1 + Number of neighbors looked up. Only 1 works for now. + """ def __init__(self, radius: int = 25, time_gap: np.timedelta64 = np.timedelta64(1, "D"), @@ -40,10 +69,30 @@ def __init__(self, radius: int = 25, @property def time_gap_seconds(self) -> float: + """``time_gap`` in seconds""" return float(self.time_gap / np.timedelta64(1, "s")) def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: - + """Match the points with the nearest node of the grid + + Invalid points have NaN values, distance and time gap in the result. Values are read + only for the valid points. + + Parameters + ---------- + dataset: xarray.Dataset + Grid with the ``lon`` and ``lat`` coordinates, and ``time`` if it has a time axis. + points: :class:`PointSet` + + Returns + ------- + :class:`MatchupResult` + + Raises + ------ + ValueError + If the grid has a time axis and the points have no time. + """ # starting by lonlat2xy on spatial grid # return FlatGrid object, flatten grid flat_grid = FlatGrid.from_grid(dataset) @@ -94,3 +143,4 @@ def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: available_methods: dict[str, type[Method]] = { "nearest": NearestNeighbor, } +"""dict: Method classes by name, used by :meth:`FloatMatcher.set_method`""" diff --git a/src/floatmatcher/product.py b/src/floatmatcher/product.py index 8f9ce41..d0e5ffc 100644 --- a/src/floatmatcher/product.py +++ b/src/floatmatcher/product.py @@ -177,13 +177,10 @@ def _open_paths(self, paths: Sequence[str]) -> xr.Dataset: def _src_dataset_checker(self, dataset: xr.Dataset) -> None: if "lon" not in dataset.coords or "lat" not in dataset.coords: - raise ValueError( - "The dataset given to GridSet object doesn't have lon or lat " - "coordinates" - ) + raise ValueError("The dataset has no lon or lat coordinates") if len(dataset.data_vars)<1: - raise ValueError("There is no variable in the dataset given to GridSet") + raise ValueError("The dataset has no variable") # test of lat/lon unicity lon = dataset["lon"].values diff --git a/src/floatmatcher/resolver.py b/src/floatmatcher/resolver.py index 61ede74..1bb5587 100644 --- a/src/floatmatcher/resolver.py +++ b/src/floatmatcher/resolver.py @@ -21,9 +21,22 @@ # ───────────────────────────────────────────────────────────── def resolve_path(root: str | Path, pattern: str, date: Any) -> str: - """Substitute a date's components into `pattern` (under `root`). available fields are - year, month, day, hour, minute (pick what the tree needs, with padding ex: - "{year}/{month:02d}/era5_{year}{month:02d}{day:02d}.nc"). + """Substitute the components of a date into a path template, under a root + + Pure helper, no disk access. The fields are ``{year}``, ``{month}``, ``{day}``, ``{hour}`` + and ``{minute}``: pick what the tree needs, with padding if wanted. + + Parameters + ---------- + root: str or pathlib.Path + pattern: str + Path template, e.g. ``"{year}/{month:02d}/era5_{year}{month:02d}{day:02d}.nc"``. + date: datetime-like + Anything :class:`pandas.Timestamp` accepts. + + Returns + ------- + str """ ts = pd.Timestamp(date) rel = pattern.format( @@ -37,16 +50,33 @@ def resolve_path(root: str | Path, pattern: str, date: Any) -> str: # ───────────────────────────────────────────────────────────── class FileResolver(ABC): - """Answers: which files must be opened for these points?""" + """Prototype of the resolvers: which files to open for these points""" @abstractmethod def files_for(self, points: PointSet | None = None) -> list[str]: + """Files to open for these points + + Parameters + ---------- + points: :class:`PointSet`, optional + Positions to match. Some resolvers do not need them. + + Returns + ------- + list of str + """ ... class ExplicitFiles(FileResolver): - """Simplest resolver: the user lists the files explicitly - or a parent directory containing all files to process. + """Files listed by the user, or found in a directory + + Parameters + ---------- + paths: str, pathlib.Path or iterable of them + Files, or directories scanned recursively. + pattern: str, default="*.nc" + Name pattern of the files looked for in a directory. """ def __init__(self, paths: str | Path | Iterable[str | Path], @@ -58,6 +88,14 @@ def __init__(self, paths: str | Path | Iterable[str | Path], self.pattern = pattern def files_for(self, points: PointSet | None = None) -> list[str]: + """Files listed, or the sorted files found in the directories + + The points are not used. + + Returns + ------- + list of str + """ resolved: list[str] = [] for entry in self._paths: path = Path(entry) @@ -73,17 +111,25 @@ def files_for(self, points: PointSet | None = None) -> list[str]: class PathTemplate(FileResolver): - """Resolve files from a declared directory pattern (ex: Datarmor). - - The user declares the shape of their tree once; for each date PRESENT in the - points, one path is built by substituting the date into the pattern. Only - the dates actually needed are resolved (no directory scan). The field type - (single-levels_inst, pressure-levels, ...) is a literal part of the pattern - chosen by the user. - - Missing files are filtered out (with a warning): the user's time tolerance - in the matchup, not the resolver, decides what gap is acceptable. But if - NOTHING resolves to an existing file, that is a configuration error -> raise. + """Files built from a path template (e.g. on Datarmor) + + One path is built for each date of the points, by substituting the date into ``pattern``. + Only the dates needed are resolved, with no directory scan. Neighboring dates are requested + too, as a point can find its nearest time step in the next or previous file. + + Missing files are skipped with a warning: the time gap of the matchup decides what is + acceptable. If no file exists at all, it is a configuration error. + + Parameters + ---------- + root: str or pathlib.Path + Root of the tree. + pattern: str + Path template under ``root``, see :func:`resolve_path`. + granularity: {'Y', 'M', 'D', 'h', 'm', 's'}, default='D' + Time step of the files, one file per step. + pad: int, default=1 + Number of neighboring steps requested on each side of every date. """ def __init__(self, root: str | Path, pattern: str, @@ -95,6 +141,29 @@ def __init__(self, root: str | Path, pattern: str, self.pad = pad def files_for(self, points: PointSet | None = None) -> list[str]: + """Existing files for the dates of the points + + Parameters + ---------- + points: :class:`PointSet` + Required: its times give the dates. + + Returns + ------- + list of str + + Raises + ------ + ValueError + If ``points`` is None or has no time. + FileNotFoundError + If none of the files exists. + + Warns + ----- + UserWarning + If the file of a date of the points is missing. + """ if points is None: raise ValueError( "PathTemplate resolves one file per date present in the points; " @@ -105,8 +174,8 @@ def files_for(self, points: PointSet | None = None) -> list[str]: np.asarray(points.time).astype(f"datetime64[{self.granularity}]") ) - # A point at 23h or 1h AM may well find its nearest step in the NEXT or PREVISOU file, so - # the neighbours of every date are requested too. + # A point at 23h or 1h AM may well find its nearest step in the NEXT or PREVIOUS file, so + # the neighbors of every date are requested too. step = np.timedelta64(1, self.granularity) wanted = dates for shift in range(1, self.pad + 1): @@ -118,7 +187,7 @@ def files_for(self, points: PointSet | None = None) -> list[str]: resolved.append(resolve_path(self.root, self.pattern, d)) # Only the dates of the requested points are worth warning about: - # padded neighbours are absent at the edges of any archive and does + # padded neighbors are absent at the edges of any archive and does # not raise warnings. asked_for = set() for d in dates: diff --git a/src/floatmatcher/sources.py b/src/floatmatcher/sources.py index 8f78a6a..707d06f 100644 --- a/src/floatmatcher/sources.py +++ b/src/floatmatcher/sources.py @@ -9,6 +9,16 @@ class Source(ABC): + """Where the files of a product come from + + A source gives the files a product needs for some points (``resolve``) and makes them + available locally (``fetch``). Subclasses implement both. + + Parameters + ---------- + path: str or list of str + Directory or files. + """ def __init__(self, path: str | Path | list[str] @@ -21,14 +31,48 @@ def __repr__(self) -> str: @abstractmethod def resolve(self, points: PointSet) -> list[str]: - """Which files are needed for these points.""" + """Files needed for these points + + Parameters + ---------- + points: :class:`PointSet` + + Returns + ------- + list of str + Paths of the files. + """ @abstractmethod def fetch(self, points: PointSet) -> None: - """Make the needed files available locally (download if remote).""" + """Make the needed files available locally, downloading them if remote + + Parameters + ---------- + points: :class:`PointSet` + """ class LocalSource(Source): + """Files already on disk + + Given as a directory, a list of files, or a root and a path template. Nothing is downloaded. + + Parameters + ---------- + path: str or list of str + Directory or files. With a ``pattern``, the root of the tree. + pattern: str, optional + Path template under ``path``, built from the dates of the points with the fields + ``{year}``, ``{month}``, ``{day}``, ``{hour}`` and ``{minute}``, for example + ``"{year}/{month:02d}/era5_{year}{month:02d}{day:02d}.nc"``. The neighboring days + are taken too. + + Raises + ------ + ValueError + If ``path`` is empty, or if a ``pattern`` is given without a path or with a list of paths. + """ def __init__(self, path: str | list[str] | Path, pattern: str | None =None) -> None: @@ -53,9 +97,23 @@ def __init__(self, def resolve(self, points: PointSet) -> list[str]: + """Files needed for these points + + A directory gives all its ``.nc`` files, a list its own files. With a ``pattern``, + one file per date of the points and of the neighboring days, the missing ones being + skipped with a warning. + + Raises + ------ + FileNotFoundError + If none of the files of a ``pattern`` exists. + ValueError + If the points have no time, with a ``pattern``. + """ return self._resolver.files_for(points) def fetch(self, points: PointSet)-> None: + """Nothing to do, the files are already on disk""" pass @@ -63,3 +121,4 @@ def fetch(self, points: PointSet)-> None: available_sources: dict[str, type[Source]] = { "local": LocalSource } +"""dict: Source classes by name, used by :class:`Product`""" diff --git a/src/floatmatcher/utils.py b/src/floatmatcher/utils.py index d397e83..1ec31c8 100644 --- a/src/floatmatcher/utils.py +++ b/src/floatmatcher/utils.py @@ -11,7 +11,24 @@ def _select_variables(ds: xr.Dataset, variables: str | list[str] | None = None) -> xr.Dataset: - """Keep only the requested variables. None -> keep all""" + """Keep only the requested variables + + Parameters + ---------- + ds: xarray.Dataset + variables: str, list of str or None + Variables to keep. None keeps all. + + Returns + ------- + :class:`xarray.Dataset` + The data are not copied. + + Raises + ------ + ValueError + If a variable is not in the dataset. + """ if variables is None: return ds @@ -26,24 +43,84 @@ def _select_variables(ds: xr.Dataset, variables: str | list[str] | None = None) def find_key(ds: xr.Dataset, name: str) -> str: + """Name of a coordinate or variable, checked to exist in the dataset + + Parameters + ---------- + ds: xarray.Dataset + name: str + + Returns + ------- + str + ``name`` itself. + + Raises + ------ + ProfileFormatError + If no coordinate or variable has this name. + """ if name in ds.coords or name in ds.variables: return name raise ProfileFormatError(f"No dataset name matches: {name}") def get(ds: xr.Dataset, name: str) -> xr.DataArray: - """Return the DataArray object""" + """DataArray of a coordinate or variable + + Parameters + ---------- + ds: xarray.Dataset + name: str + + Returns + ------- + :class:`xarray.DataArray` + + Raises + ------ + ProfileFormatError + If no coordinate or variable has this name. + """ return ds[find_key(ds, name)] def extract(ds: xr.Dataset, name: str) -> NDArray[Any]: - """Return the .values (raw ndarray) — the common case.""" + """Values of a coordinate or variable, as a NumPy array + + Parameters + ---------- + ds: xarray.Dataset + name: str + + Returns + ------- + numpy.ndarray + + Raises + ------ + ProfileFormatError + If no coordinate or variable has this name. + """ return get(ds, name).values def indented_repr(obj: object, indent: int = 4) -> str: - """Indented listing of obj attributes. Attributes that are floatmatcher - objects are listed one level deeper.""" + """Indented listing of the attributes of an object + + Attributes that are floatmatcher objects are listed one level deeper. Used by the + ``__repr__`` of the products, sources and methods. + + Parameters + ---------- + obj: object + indent: int, default=4 + Number of spaces of the first level. + + Returns + ------- + str + """ pad = " " * indent text = f"\n{pad}type: {type(obj).__name__}" for name, value in obj.__dict__.items(): diff --git a/tests/helpers.py b/tests/helpers.py index 0fdf7e6..b16611c 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -15,7 +15,7 @@ def daily_timestamps(n): return set_timestamps(*range(1, n + 1)) def make_grid(lat, lon, time=None, variables=("v",)): - """Build a GridSet-ready Dataset filled with zeros. + """Build a grid Dataset (lat, lon[, time]) filled with zeros. """ lat = np.asarray(lat, dtype=float) lon = np.asarray(lon, dtype=float) diff --git a/tests/test_products.py b/tests/test_products.py index e1a500c..69a46df 100644 --- a/tests/test_products.py +++ b/tests/test_products.py @@ -53,7 +53,7 @@ def test_rename_bare_dimension(): def test_data_variable_is_promoted_to_coord(): """lon/lat carried as data variables (anonymous dims) become coordinates: - GridSet reads them from ds.coords, never from ds.data_vars.""" + the product reads them from ds.coords, never from ds.data_vars.""" ds = xr.Dataset( { "sst": (("y", "x"), np.zeros((2, 2))), From 112a74e052bd3df348ba969a7517103ed073fa37 Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Mon, 12 Oct 2026 12:26:00 +0000 Subject: [PATCH 08/10] docstrings private functions --- src/floatmatcher/flatgrid.py | 14 ++++++-- src/floatmatcher/floatmatcher.py | 13 ++++++-- src/floatmatcher/matchup_results.py | 3 +- src/floatmatcher/methods.py | 2 +- src/floatmatcher/product.py | 50 +++++++++++++++++++++++++++-- src/floatmatcher/sources.py | 16 +++++++-- 6 files changed, 85 insertions(+), 13 deletions(-) diff --git a/src/floatmatcher/flatgrid.py b/src/floatmatcher/flatgrid.py index 5580398..8adc16d 100644 --- a/src/floatmatcher/flatgrid.py +++ b/src/floatmatcher/flatgrid.py @@ -103,7 +103,8 @@ def spatial_nearest(self, points: PointSet, Returns ------- tuple of arrays - Distances (km) and indices of the nodes. + Distances (km) and indices of the nodes. Distances are straight lines between + the points on the sphere, close to the great-circle ones at short range. """ dist: NDArray[np.float64] idx: NDArray[np.int64] @@ -142,10 +143,19 @@ def temporal_nearest(self, points: PointSet, def _to_seconds(times: NDArray[np.datetime64]) -> NDArray[np.float64]: - """Convert datetime64 to floating-point seconds since a fixed epoch. + """Convert datetime64 to floating-point seconds since a fixed epoch Working in a common float unit lets the 1D KDTree measure time distance, and returning *seconds* makes the time_gap_seconds constraint directly comparable. + + Parameters + ---------- + times: numpy.ndarray of datetime64 + + Returns + ------- + numpy.ndarray of float + Seconds since ``REF_TIME``. """ delta = np.asarray(times, dtype=f"datetime64[{TIME_UNIT}]") - REF_TIME seconds: NDArray[np.float64] = delta / np.timedelta64(1, "s") diff --git a/src/floatmatcher/floatmatcher.py b/src/floatmatcher/floatmatcher.py index f848e57..c0eb032 100644 --- a/src/floatmatcher/floatmatcher.py +++ b/src/floatmatcher/floatmatcher.py @@ -85,6 +85,8 @@ def set_points(self, *data: Any, **names: str) -> None: ------ TypeError If the input is not supported, or if names are given with arrays or a PointSet. + ValueError + If lon, lat and time do not have the same length. ProfileFormatError If the lon or lat column or variable is missing (DataFrame or Dataset). @@ -135,7 +137,8 @@ def set_product(self, Raises ------ ValueError - If the product is unknown, or the source is not available for it. + If the product is unknown, the source is not available for it, or + ``selected_variables`` is neither None nor a non-empty list. TypeError If a source parameter is unknown. """ @@ -190,12 +193,16 @@ def result(self) -> MatchupResult: def match(self) -> None: """Run the matchup - Read the result with ``result``. + Opens the product on the files needed by the points, then applies the method. Read the + result with ``result``. Raises ------ ValueError - If the points, the product or the method are not set. + If the points, the product or the method are not set, if the product dataset fails + a check, or if the points have no time and the grid has a time axis. + FileNotFoundError + If no file of the product is found for the points. """ if self.points is None: raise ValueError("match(): no points set, call set_points(...)") diff --git a/src/floatmatcher/matchup_results.py b/src/floatmatcher/matchup_results.py index fa46374..66d45d1 100644 --- a/src/floatmatcher/matchup_results.py +++ b/src/floatmatcher/matchup_results.py @@ -21,7 +21,8 @@ class MatchupResult: values: dict Matched values, one array per variable. distance_km: numpy.ndarray - Distance to the matched node, in km. + Distance to the matched node, in km (straight line on the sphere, close to the + great-circle distance at short range). time_delta: numpy.ndarray Gap to the matched time step, in seconds. NaN for a grid without time axis. valid: numpy.ndarray of bool diff --git a/src/floatmatcher/methods.py b/src/floatmatcher/methods.py index 3ea16ce..c802166 100644 --- a/src/floatmatcher/methods.py +++ b/src/floatmatcher/methods.py @@ -53,7 +53,7 @@ class NearestNeighbor(Method): Parameters ---------- radius: int, default=25 - Maximum distance to the node, in km. + Maximum distance to the node, in km (straight line on the sphere). time_gap: numpy.timedelta64, default=1 day Maximum gap to the time step. Ignored for a grid without time axis. k_nearest: int, default=1 diff --git a/src/floatmatcher/product.py b/src/floatmatcher/product.py index d0e5ffc..a303da1 100644 --- a/src/floatmatcher/product.py +++ b/src/floatmatcher/product.py @@ -157,12 +157,26 @@ def available_variables(self) -> list[str]: def _open_paths(self, paths: Sequence[str]) -> xr.Dataset: - """ open several files as a xr mf_dataset with options - // lock option fix made by AI to avoid segfault : open_mfdataset + """Open several files as one lazy dataset + + // lock option fix made by AI to avoid segfault : open_mfdataset returns dask arrays read by multiple threads, but netCDF4/HDF5 isn't thread-safe here: concurrent access to xarray's file cache corrupted memory and killed the process. A single shared lock serialises those reads // + + Parameters + ---------- + paths: list of str + + Returns + ------- + :class:`xarray.Dataset` + + Raises + ------ + ValueError + If ``paths`` is empty. """ if not paths: raise ValueError("empty file list") @@ -176,6 +190,20 @@ def _open_paths(self, paths: Sequence[str]) -> xr.Dataset: ) def _src_dataset_checker(self, dataset: xr.Dataset) -> None: + """Check that a normalized dataset can be matched as a grid + + It needs the lon and lat coordinates, at least one variable, and no duplicated + longitude or latitude + + Parameters + ---------- + dataset: xarray.Dataset + + Raises + ------ + ValueError + If a check fails. + """ if "lon" not in dataset.coords or "lat" not in dataset.coords: raise ValueError("The dataset has no lon or lat coordinates") @@ -191,6 +219,17 @@ def _src_dataset_checker(self, dataset: xr.Dataset) -> None: raise ValueError("grid: duplicated latitudes in array") def _check_time_unicity(self, dataset: xr.Dataset) -> None: + """Check that the time steps are unique + + Parameters + ---------- + dataset: xarray.Dataset + + Raises + ------ + ValueError + If a timestamp is duplicated. + """ # a grid without time axis (2D) has nothing to check if "time" in dataset.coords: times = dataset["time"].values @@ -212,6 +251,8 @@ def open(self, points: PointSet) -> None: ------ ValueError If there is no file, or if the dataset fails a check. + FileNotFoundError + If none of the files of a ``pattern`` exists. """ files = self._source.resolve(points) ds = self._open_paths(files) @@ -250,6 +291,7 @@ class ERA5Product(Product): src_available = ["local"] def normalize(self, ds_raw: xr.Dataset) -> xr.Dataset: + """Set lon and lat as coordinates, harmonize names""" ds = to_standard(ds_raw, self.coord_map) return ds @@ -263,6 +305,7 @@ class LUTProduct(Product): src_available = ["local"] def normalize(self, ds_raw: xr.Dataset) -> xr.Dataset: + """Set lon and lat as coordinates, harmonize names""" ds = to_standard(ds_raw, self.coord_map) return ds @@ -274,4 +317,5 @@ def normalize(self, ds_raw: xr.Dataset) -> xr.Dataset: available_products: dict[str, type[Product]] = { "era5": ERA5Product, "lut" : LUTProduct -} \ No newline at end of file +} +"""dict: Product classes by name, used by :meth:`FloatMatcher.set_product`""" \ No newline at end of file diff --git a/src/floatmatcher/sources.py b/src/floatmatcher/sources.py index 707d06f..f7b80ca 100644 --- a/src/floatmatcher/sources.py +++ b/src/floatmatcher/sources.py @@ -99,9 +99,19 @@ def __init__(self, def resolve(self, points: PointSet) -> list[str]: """Files needed for these points - A directory gives all its ``.nc`` files, a list its own files. With a ``pattern``, - one file per date of the points and of the neighboring days, the missing ones being - skipped with a warning. + A directory gives all its ``.nc`` files, recursively; a list gives its own files. With a + ``pattern``, one file per date of the points and of the neighboring days. Missing files + are skipped, with a warning for the dates of the points only (neighboring days are + often absent at the edges of an archive). + + Parameters + ---------- + points: :class:`PointSet` + + Returns + ------- + list of str + Paths of the files. Raises ------ From b835c3def3fb52aec4be2e9cec97f9b63c9b3d9e Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Mon, 12 Oct 2026 13:09:06 +0000 Subject: [PATCH 09/10] switch public functions to private --- src/floatmatcher/methods.py | 4 ++-- src/floatmatcher/pointset.py | 12 ++++++------ src/floatmatcher/product.py | 4 ++-- src/floatmatcher/resolver.py | 9 +++++---- src/floatmatcher/sources.py | 4 ++-- src/floatmatcher/utils.py | 14 +++++++------- tests/test_pointset.py | 14 +++++++------- tests/test_resolver.py | 6 +++--- 8 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/floatmatcher/methods.py b/src/floatmatcher/methods.py index c802166..632c985 100644 --- a/src/floatmatcher/methods.py +++ b/src/floatmatcher/methods.py @@ -12,7 +12,7 @@ from .flatgrid import FlatGrid from .matchup_results import MatchupResult from .pointset import PointSet -from .utils import indented_repr +from .utils import _indented_repr class Method(ABC): @@ -24,7 +24,7 @@ class Method(ABC): """ def __repr__(self) -> str: - return indented_repr(self) + return _indented_repr(self) @abstractmethod def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: diff --git a/src/floatmatcher/pointset.py b/src/floatmatcher/pointset.py index 2d4c3db..e7a66fe 100644 --- a/src/floatmatcher/pointset.py +++ b/src/floatmatcher/pointset.py @@ -11,7 +11,7 @@ from .exceptions import ProfileFormatError from .geo import lonlat_to_xyz from .constants import TIME_UNIT -from .utils import extract, get +from .utils import _extract, _get @dataclass(repr=False) @@ -59,9 +59,9 @@ def __post_init__(self) -> None: self.lat = np.asarray(self.lat, dtype=float) if self.time is not None: self.time = np.asarray(self.time, dtype=f"datetime64[{TIME_UNIT}]") - self.check_lengths() + self._check_lengths() - def check_lengths(self) -> None: + def _check_lengths(self) -> None: """Raise ValueError if lon, lat and time (when set) differ in length""" if self.time is not None: lengths = [len(self.lon), len(self.lat), len(self.time)] @@ -196,17 +196,17 @@ def from_xrdataset(cls, ds: xr.Dataset, lon: str = "LONGITUDE", If neither the time variable nor ``"JULD"`` exists. """ # The point-dimension name is DISCOVERED (da_lon.dims[0]), never assumed. - da_lon = get(ds, lon) + da_lon = _get(ds, lon) point_dim = da_lon.dims[0] # DISCOVERED, never assumed # 2. extract the three arrays as ndarrays lon_arr = da_lon.values - lat_arr = extract(ds, lat) + lat_arr = _extract(ds, lat) time_arr = None if time is not None: for name in [time, "JULD"]: if name in ds.coords or name in ds.variables: - time_arr = extract(ds, name) + time_arr = _extract(ds, name) break if time_arr is None: warnings.warn(f"No variable named {time!r} or 'JULD': the points have no time. " diff --git a/src/floatmatcher/product.py b/src/floatmatcher/product.py index a303da1..dfd5ed9 100644 --- a/src/floatmatcher/product.py +++ b/src/floatmatcher/product.py @@ -7,7 +7,7 @@ from pathlib import Path from .sources import available_sources -from .utils import indented_repr, _select_variables +from .utils import _indented_repr, _select_variables from .pointset import PointSet @@ -263,7 +263,7 @@ def open(self, points: PointSet) -> None: def __repr__(self) -> str: - return indented_repr(self) + return _indented_repr(self) @abstractmethod def normalize(self, ds_raw: xr.Dataset) -> xr.Dataset: diff --git a/src/floatmatcher/resolver.py b/src/floatmatcher/resolver.py index 1bb5587..46b1f87 100644 --- a/src/floatmatcher/resolver.py +++ b/src/floatmatcher/resolver.py @@ -20,7 +20,7 @@ # Path resolution (pure helper, no disk access) # ───────────────────────────────────────────────────────────── -def resolve_path(root: str | Path, pattern: str, date: Any) -> str: +def _resolve_path(root: str | Path, pattern: str, date: Any) -> str: """Substitute the components of a date into a path template, under a root Pure helper, no disk access. The fields are ``{year}``, ``{month}``, ``{day}``, ``{hour}`` @@ -125,7 +125,8 @@ class PathTemplate(FileResolver): root: str or pathlib.Path Root of the tree. pattern: str - Path template under ``root``, see :func:`resolve_path`. + Path template under ``root``, with the fields ``{year}``, ``{month}``, ``{day}``, + ``{hour}`` and ``{minute}``. granularity: {'Y', 'M', 'D', 'h', 'm', 's'}, default='D' Time step of the files, one file per step. pad: int, default=1 @@ -184,14 +185,14 @@ def files_for(self, points: PointSet | None = None) -> list[str]: resolved = [] for d in wanted: - resolved.append(resolve_path(self.root, self.pattern, d)) + resolved.append(_resolve_path(self.root, self.pattern, d)) # Only the dates of the requested points are worth warning about: # padded neighbors are absent at the edges of any archive and does # not raise warnings. asked_for = set() for d in dates: - asked_for.add(resolve_path(self.root, self.pattern, d)) + asked_for.add(_resolve_path(self.root, self.pattern, d)) existing = [] missing = [] diff --git a/src/floatmatcher/sources.py b/src/floatmatcher/sources.py index f7b80ca..a43da48 100644 --- a/src/floatmatcher/sources.py +++ b/src/floatmatcher/sources.py @@ -5,7 +5,7 @@ from .pointset import PointSet from .resolver import FileResolver, PathTemplate, ExplicitFiles -from .utils import indented_repr +from .utils import _indented_repr class Source(ABC): @@ -27,7 +27,7 @@ def __init__(self, def __repr__(self) -> str: - return indented_repr(self) + return _indented_repr(self) @abstractmethod def resolve(self, points: PointSet) -> list[str]: diff --git a/src/floatmatcher/utils.py b/src/floatmatcher/utils.py index 1ec31c8..7b0ce4c 100644 --- a/src/floatmatcher/utils.py +++ b/src/floatmatcher/utils.py @@ -42,7 +42,7 @@ def _select_variables(ds: xr.Dataset, variables: str | list[str] | None = None) return ds[wanted] -def find_key(ds: xr.Dataset, name: str) -> str: +def _find_key(ds: xr.Dataset, name: str) -> str: """Name of a coordinate or variable, checked to exist in the dataset Parameters @@ -65,7 +65,7 @@ def find_key(ds: xr.Dataset, name: str) -> str: raise ProfileFormatError(f"No dataset name matches: {name}") -def get(ds: xr.Dataset, name: str) -> xr.DataArray: +def _get(ds: xr.Dataset, name: str) -> xr.DataArray: """DataArray of a coordinate or variable Parameters @@ -82,10 +82,10 @@ def get(ds: xr.Dataset, name: str) -> xr.DataArray: ProfileFormatError If no coordinate or variable has this name. """ - return ds[find_key(ds, name)] + return ds[_find_key(ds, name)] -def extract(ds: xr.Dataset, name: str) -> NDArray[Any]: +def _extract(ds: xr.Dataset, name: str) -> NDArray[Any]: """Values of a coordinate or variable, as a NumPy array Parameters @@ -102,10 +102,10 @@ def extract(ds: xr.Dataset, name: str) -> NDArray[Any]: ProfileFormatError If no coordinate or variable has this name. """ - return get(ds, name).values + return _get(ds, name).values -def indented_repr(obj: object, indent: int = 4) -> str: +def _indented_repr(obj: object, indent: int = 4) -> str: """Indented listing of the attributes of an object Attributes that are floatmatcher objects are listed one level deeper. Used by the @@ -125,7 +125,7 @@ def indented_repr(obj: object, indent: int = 4) -> str: text = f"\n{pad}type: {type(obj).__name__}" for name, value in obj.__dict__.items(): if type(value).__module__.startswith("floatmatcher"): - text += f"\n{pad}{name}:{indented_repr(value, indent + 4)}" + text += f"\n{pad}{name}:{_indented_repr(value, indent + 4)}" else: text += f"\n{pad}{name}: {value}" return text diff --git a/tests/test_pointset.py b/tests/test_pointset.py index 1080090..2f549f8 100644 --- a/tests/test_pointset.py +++ b/tests/test_pointset.py @@ -9,7 +9,7 @@ from floatmatcher.exceptions import ProfileFormatError from floatmatcher.pointset import PointSet -from floatmatcher.utils import extract, find_key, get +from floatmatcher.utils import _extract, _find_key, _get from helpers import daily_timestamps @@ -127,23 +127,23 @@ def df_datetime_index(): ) -# ---------- Helpers: find_key / get / extract ---------- +# ---------- Helpers: _find_key / _get / _extract ---------- def test_find_key_matches_variable_or_coord(argopy_like_ds): # variable match - assert find_key(argopy_like_ds, "LONGITUDE") == "LONGITUDE" + assert _find_key(argopy_like_ds, "LONGITUDE") == "LONGITUDE" # coordinate match (N_PROF is a coord here) - assert find_key(argopy_like_ds, "N_PROF") == "N_PROF" + assert _find_key(argopy_like_ds, "N_PROF") == "N_PROF" def test_find_key_raises_when_no_candidate_matches(argopy_like_ds): with pytest.raises(ProfileFormatError): - find_key(argopy_like_ds, "DUMMY") + _find_key(argopy_like_ds, "DUMMY") def test_get_returns_dataarray_extract_returns_ndarray(argopy_like_ds): - da = get(argopy_like_ds, "LONGITUDE") + da = _get(argopy_like_ds, "LONGITUDE") assert isinstance(da, xr.DataArray) # object, has .dims - arr = extract(argopy_like_ds, "LONGITUDE") + arr = _extract(argopy_like_ds, "LONGITUDE") assert isinstance(arr, np.ndarray) # raw values diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 9d4e0d9..31cdef4 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -8,7 +8,7 @@ import numpy as np import pytest -from floatmatcher.resolver import resolve_path, ExplicitFiles, PathTemplate +from floatmatcher.resolver import _resolve_path, ExplicitFiles, PathTemplate from floatmatcher.pointset import PointSet @@ -40,10 +40,10 @@ def _pts(*timestamps): time=np.array(timestamps, dtype="datetime64[ns]")) -# ---------- resolve_path (pure, no disk) ---------- +# ---------- _resolve_path (pure, no disk) ---------- def test_resolve_path(): - p = resolve_path("/data", PATTERN, np.datetime64("2015-01-01")) + p = _resolve_path("/data", PATTERN, np.datetime64("2015-01-01")) assert Path(p) == Path("/data/2015/01/era5_20150101.nc") From 9572a56e3f6d72c89e74f5e4122976c0ab137613 Mon Sep 17 00:00:00 2001 From: Clement WEBER Date: Mon, 12 Oct 2026 13:39:59 +0000 Subject: [PATCH 10/10] setup sphinx v1 --- .gitignore | 4 +- docs/_templates/autosummary/base.rst | 5 ++ docs/_templates/autosummary/class.rst | 7 +++ docs/api.rst | 77 +++++++++++++++++++++++++++ docs/conf.py | 45 ++++++++++++++++ docs/index.rst | 25 +++++++++ pyproject.toml | 6 +++ src/floatmatcher/__init__.py | 9 +++- src/floatmatcher/flatgrid.py | 6 +-- src/floatmatcher/floatmatcher.py | 2 +- src/floatmatcher/matchup_results.py | 4 +- src/floatmatcher/methods.py | 12 ++--- src/floatmatcher/pointset.py | 6 +-- src/floatmatcher/product.py | 4 +- src/floatmatcher/resolver.py | 4 +- src/floatmatcher/sources.py | 8 +-- tests/test_basic.py | 8 ++- 17 files changed, 205 insertions(+), 27 deletions(-) create mode 100644 docs/_templates/autosummary/base.rst create mode 100644 docs/_templates/autosummary/class.rst create mode 100644 docs/api.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst diff --git a/.gitignore b/.gitignore index 60749c8..206aa22 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,8 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +.ruff_cache/ +.mypy_cache/ # Translations *.mo @@ -80,7 +82,7 @@ instance/ # Sphinx documentation docs/_build/ -docs/ +docs/generated/ # PyBuilder target/ diff --git a/docs/_templates/autosummary/base.rst b/docs/_templates/autosummary/base.rst new file mode 100644 index 0000000..3fe9858 --- /dev/null +++ b/docs/_templates/autosummary/base.rst @@ -0,0 +1,5 @@ +{{ objname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. auto{{ objtype }}:: {{ objname }} diff --git a/docs/_templates/autosummary/class.rst b/docs/_templates/autosummary/class.rst new file mode 100644 index 0000000..d3a5e37 --- /dev/null +++ b/docs/_templates/autosummary/class.rst @@ -0,0 +1,7 @@ +{{ objname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :show-inheritance: diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..3aa70ec --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,77 @@ +API reference +============= + +This page is generated from the docstrings. + +Matchup +------- + +.. currentmodule:: floatmatcher.floatmatcher + +.. autosummary:: + :toctree: generated/ + + FloatMatcher + +Points and results +------------------ + +.. currentmodule:: floatmatcher.pointset + +.. autosummary:: + :toctree: generated/ + + PointSet + +.. currentmodule:: floatmatcher.matchup_results + +.. autosummary:: + :toctree: generated/ + + MatchupResult + +Errors +------ + +.. currentmodule:: floatmatcher.exceptions + +.. autosummary:: + :toctree: generated/ + + FloatMatcherError + ProfileFormatError + +Extending FloatMatcher +---------------------- + +Products, sources and methods are classes registered by name, so that ``set_product`` and +``set_method`` can find them. + +.. currentmodule:: floatmatcher.product + +.. autosummary:: + :toctree: generated/ + + Product + ERA5Product + LUTProduct + to_standard + available_products + +.. currentmodule:: floatmatcher.sources + +.. autosummary:: + :toctree: generated/ + + Source + LocalSource + available_sources + +.. currentmodule:: floatmatcher.methods + +.. autosummary:: + :toctree: generated/ + + Method + NearestNeighbor + available_methods diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..98b0f0a --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,45 @@ +# Configuration file for the Sphinx documentation builder. +# Build: uv run --group docs sphinx-build -W -b html docs docs/_build/html + +import floatmatcher + +project = "FloatMatcher" +author = "Clement Weber, Guillaume Maze, Kevin Ballem, Alban Sizun, Virginie Racapé" +copyright = "EUPL v1.2" +release = floatmatcher.__version__ +version = release + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "numpydoc", + "sphinx_copybutton", +] + +templates_path = ["_templates"] +exclude_patterns = ["_build"] + +# API pages are generated from the docstrings (numpydoc style, like argopy) +autosummary_generate = True +numpydoc_show_class_members = False + +# readable names: "FloatMatcher" rather than "floatmatcher.floatmatcher.FloatMatcher" +add_module_names = False # signatures +toc_object_entries_show_parents = "hide" # "On this page": set_points, not FloatMatcher.set_points +autodoc_member_order = "bysource" # methods in the order of the code (the user workflow) + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable", None), + "pandas": ("https://pandas.pydata.org/docs", None), + "xarray": ("https://docs.xarray.dev/en/stable", None), +} + +html_theme = "sphinx_book_theme" +html_title = "FloatMatcher" +html_theme_options = { + "repository_url": "https://github.com/euroargodev/FloatMatcher", + "use_repository_button": True, +} diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..01dbcb5 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,25 @@ +FloatMatcher +============ + +FloatMatcher matches positions, such as Argo profiles, with a gridded product such as the +ERA5 reanalysis: each position gets the values of the nearest node of the grid, within a +distance and a time gap. + +.. code-block:: python + + import numpy as np + from floatmatcher.floatmatcher import FloatMatcher + + fm = FloatMatcher() + fm.set_points(lon, lat, time) # or a DataFrame, or an xarray Dataset + fm.set_product("era5", "local", "/data/era5", ["sst"], + pattern="{year}/{month:02d}/era5_{year}{month:02d}{day:02d}.nc") + fm.set_method("nearest", radius=50, time_gap=np.timedelta64(3, "h")) + fm.match() + fm.result # matched values, distances and time gaps + fm.reinject() # the input DataFrame or Dataset, with _coloc added + +.. toctree:: + :maxdepth: 2 + + api diff --git a/pyproject.toml b/pyproject.toml index e9cd63c..692d32e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,12 @@ dev = [ "mypy", "ruff", ] +docs = [ + "numpydoc>=1.11.0", + "sphinx>=9.0.4", + "sphinx-book-theme>=1.4.0", + "sphinx-copybutton>=0.5.2", +] [tool.hatch.build.targets.wheel] packages = ["src/floatmatcher"] diff --git a/src/floatmatcher/__init__.py b/src/floatmatcher/__init__.py index 3dc1f76..14d30aa 100644 --- a/src/floatmatcher/__init__.py +++ b/src/floatmatcher/__init__.py @@ -1 +1,8 @@ -__version__ = "0.1.0" +from importlib.metadata import PackageNotFoundError, version + +try: + # single source of truth: the version of pyproject.toml, read from the installed package + __version__ = version("floatmatcher") +except PackageNotFoundError: + # local copy, not installed (same fallback as argopy) + __version__ = "999" diff --git a/src/floatmatcher/flatgrid.py b/src/floatmatcher/flatgrid.py index 8adc16d..e8df5b0 100644 --- a/src/floatmatcher/flatgrid.py +++ b/src/floatmatcher/flatgrid.py @@ -55,7 +55,7 @@ def from_grid(cls, ds: xr.Dataset) -> "FlatGrid": Returns ------- - :class:`FlatGrid` + :class:`.FlatGrid` """ stacked = ds.stack(node=("lat", "lon")) time = ds["time"].values if "time" in ds.coords else None @@ -96,7 +96,7 @@ def spatial_nearest(self, points: PointSet, Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` k: int, default=1 Number of neighbors. @@ -117,7 +117,7 @@ def temporal_nearest(self, points: PointSet, Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` k: int, default=1 Number of neighbors. diff --git a/src/floatmatcher/floatmatcher.py b/src/floatmatcher/floatmatcher.py index c0eb032..485f64d 100644 --- a/src/floatmatcher/floatmatcher.py +++ b/src/floatmatcher/floatmatcher.py @@ -76,7 +76,7 @@ def set_points(self, *data: Any, **names: str) -> None: ---------- *data ``lon, lat[, time]`` arrays, a :class:`pandas.DataFrame`, a :class:`xarray.Dataset` or a - :class:`PointSet`. + :class:`.PointSet`. **names: str ``lon``, ``lat``, ``time``: names of the columns or variables holding them. Only for a DataFrame or a Dataset. diff --git a/src/floatmatcher/matchup_results.py b/src/floatmatcher/matchup_results.py index 66d45d1..4edc127 100644 --- a/src/floatmatcher/matchup_results.py +++ b/src/floatmatcher/matchup_results.py @@ -48,7 +48,7 @@ def _to_dataset(self, points: PointSet) -> xr.Dataset: Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` Returns ------- @@ -87,7 +87,7 @@ def _to_dataframe(self, points: PointSet) -> pd.DataFrame: Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` Returns ------- diff --git a/src/floatmatcher/methods.py b/src/floatmatcher/methods.py index 632c985..ff68bf1 100644 --- a/src/floatmatcher/methods.py +++ b/src/floatmatcher/methods.py @@ -19,7 +19,7 @@ class Method(ABC): """Prototype of the matchup methods A method holds its parameters and matches points with a grid in ``apply``. It knows - nothing of the product or of :class:`FloatMatcher`: it receives the opened grid and the + nothing of the product or of :class:`.FloatMatcher`: it receives the opened grid and the points. Subclasses implement ``apply`` and are registered in ``available_methods``. """ @@ -34,11 +34,11 @@ def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: ---------- dataset: xarray.Dataset Grid with the ``lon`` and ``lat`` coordinates, and ``time`` if it has a time axis. - points: :class:`PointSet` + points: :class:`.PointSet` Returns ------- - :class:`MatchupResult` + :class:`.MatchupResult` """ ... @@ -82,11 +82,11 @@ def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: ---------- dataset: xarray.Dataset Grid with the ``lon`` and ``lat`` coordinates, and ``time`` if it has a time axis. - points: :class:`PointSet` + points: :class:`.PointSet` Returns ------- - :class:`MatchupResult` + :class:`.MatchupResult` Raises ------ @@ -143,4 +143,4 @@ def apply(self, dataset: xr.Dataset, points: PointSet) -> MatchupResult: available_methods: dict[str, type[Method]] = { "nearest": NearestNeighbor, } -"""dict: Method classes by name, used by :meth:`FloatMatcher.set_method`""" +"""dict: Method classes by name, used by :meth:`.FloatMatcher.set_method`""" diff --git a/src/floatmatcher/pointset.py b/src/floatmatcher/pointset.py index e7a66fe..ee725de 100644 --- a/src/floatmatcher/pointset.py +++ b/src/floatmatcher/pointset.py @@ -103,7 +103,7 @@ def from_arrays(cls, Returns ------- - :class:`PointSet` + :class:`.PointSet` """ # Nothing to extract, no origin_* (bare arrays). Converted in __post_init__. return cls(lon, lat, time) # type: ignore[arg-type] @@ -129,7 +129,7 @@ def from_dataframe(cls, df: pd.DataFrame | pd.Series, lon: str = "longitude", Returns ------- - :class:`PointSet` + :class:`.PointSet` Raises ------ @@ -183,7 +183,7 @@ def from_xrdataset(cls, ds: xr.Dataset, lon: str = "LONGITUDE", Returns ------- - :class:`PointSet` + :class:`.PointSet` Raises ------ diff --git a/src/floatmatcher/product.py b/src/floatmatcher/product.py index dfd5ed9..cb5752e 100644 --- a/src/floatmatcher/product.py +++ b/src/floatmatcher/product.py @@ -244,7 +244,7 @@ def open(self, points: PointSet) -> None: Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` Positions to match, they decide which files are needed. Raises @@ -318,4 +318,4 @@ def normalize(self, ds_raw: xr.Dataset) -> xr.Dataset: "era5": ERA5Product, "lut" : LUTProduct } -"""dict: Product classes by name, used by :meth:`FloatMatcher.set_product`""" \ No newline at end of file +"""dict: Product classes by name, used by :meth:`.FloatMatcher.set_product`""" \ No newline at end of file diff --git a/src/floatmatcher/resolver.py b/src/floatmatcher/resolver.py index 46b1f87..d8d0fdf 100644 --- a/src/floatmatcher/resolver.py +++ b/src/floatmatcher/resolver.py @@ -58,7 +58,7 @@ def files_for(self, points: PointSet | None = None) -> list[str]: Parameters ---------- - points: :class:`PointSet`, optional + points: :class:`.PointSet`, optional Positions to match. Some resolvers do not need them. Returns @@ -146,7 +146,7 @@ def files_for(self, points: PointSet | None = None) -> list[str]: Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` Required: its times give the dates. Returns diff --git a/src/floatmatcher/sources.py b/src/floatmatcher/sources.py index a43da48..c9722f0 100644 --- a/src/floatmatcher/sources.py +++ b/src/floatmatcher/sources.py @@ -35,7 +35,7 @@ def resolve(self, points: PointSet) -> list[str]: Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` Returns ------- @@ -49,7 +49,7 @@ def fetch(self, points: PointSet) -> None: Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` """ @@ -106,7 +106,7 @@ def resolve(self, points: PointSet) -> list[str]: Parameters ---------- - points: :class:`PointSet` + points: :class:`.PointSet` Returns ------- @@ -131,4 +131,4 @@ def fetch(self, points: PointSet)-> None: available_sources: dict[str, type[Source]] = { "local": LocalSource } -"""dict: Source classes by name, used by :class:`Product`""" +"""dict: Source classes by name, used by :class:`.Product`""" diff --git a/tests/test_basic.py b/tests/test_basic.py index e6f364e..211da35 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,5 +1,9 @@ +import tomllib +from pathlib import Path + from floatmatcher import __version__ -def test_version() -> None: - assert __version__ == "0.1.0" +def test_version_comes_from_pyproject() -> None: + pyproject = tomllib.loads((Path(__file__).parents[1] / "pyproject.toml").read_text()) + assert __version__ == pyproject["project"]["version"]