From 17c982aacd13c0c785be8b400be48379593b1014 Mon Sep 17 00:00:00 2001 From: Michael Niklas Date: Tue, 29 Sep 2026 01:26:11 +0200 Subject: [PATCH] Fix nightly upstream-dev failures (#11501) (#11627) * Fix nightly upstream-dev failures (#11501) - Propagate NaN in min/max of object arrays with skipna=False instead of returning a result that depends on the position of the missing value (now matches pandas dev behaviour) - Avoid the deprecated Series.values for timezone-aware Series in as_compatible_data - Skip TestDask::test_roundtrip_stringdtype_nulls: the dask "roundtrip" does not encode, so nulls are never replaced; it only passed because numpy < 2.6 compared "" == None as True for StringDType * Fix upstream-dev mypy --------- Co-authored-by: Claude Co-authored-by: Deepak Cherian --- .pre-commit-config.yaml | 6 +++--- ci/nightly/pixi.toml | 2 +- doc/whats-new.rst | 7 +++++++ xarray/core/duck_array_ops.py | 12 ++++++++++++ xarray/core/variable.py | 4 ++++ xarray/tests/test_backends.py | 6 ++++++ xarray/tests/test_dataarray.py | 10 ++++------ xarray/tests/test_duck_array_ops.py | 21 ++++++++++++++++++++- xarray/tests/test_plot.py | 8 +++++--- xarray/tests/test_variable.py | 2 +- 10 files changed, 63 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2f30c76367..27dacded6dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,14 +67,14 @@ repos: - id: sync-mypy-version name: sync mypy version language: python - files: ^\.pre-commit-config\.yaml$ + files: ^(\.pre-commit-config\.yaml|pixi\.toml|ci/nightly/pixi\.toml)$ pass_filenames: false entry: >- python -c "import pathlib, re; config = pathlib.Path('.pre-commit-config.yaml').read_text(); version = re.search(r'mirrors-mypy\s+rev: v?([0-9.]+)', config).group(1); - pixi = pathlib.Path('pixi.toml'); - pixi.write_text(re.sub(r'(?m)^mypy = \"==[0-9.]+\"', f'mypy = \"=={version}\"', pixi.read_text())); + [pixi.write_text(re.sub(r'(?m)^mypy = \"==[0-9.]+\"', f'mypy = \"=={version}\"', pixi.read_text())) + for pixi in map(pathlib.Path, ['pixi.toml', 'ci/nightly/pixi.toml'])]; pyproject = pathlib.Path('pyproject.toml'); pyproject.write_text(re.sub(r'\"mypy==[0-9.]+\"', f'\"mypy=={version}\"', pyproject.read_text()))" - repo: https://github.com/citation-file-format/cff-converter-python diff --git a/ci/nightly/pixi.toml b/ci/nightly/pixi.toml index 0f4f640c95e..16d7d99b174 100644 --- a/ci/nightly/pixi.toml +++ b/ci/nightly/pixi.toml @@ -80,7 +80,7 @@ cartopy = "*" seaborn = "*" [feature.typing.dependencies] -mypy = "==1.19.1" +mypy = "==2.3.1" pyright = "*" hypothesis = "*" lxml = "*" diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4cf481f618e..d03dee7ce42 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -53,6 +53,13 @@ Deprecations Bug Fixes ~~~~~~~~~ +- ``min`` and ``max`` of object arrays with ``skipna=False`` now return ``NaN`` + for slices containing missing values, instead of a result that depended on + the position of the missing value (:issue:`11501`, :pull:`#11627`). + By `Michael Niklas `_. +- Avoid pandas' deprecated ``Series.values`` when creating a + :py:class:`~xarray.Variable` from a timezone-aware :py:class:`pandas.Series` + (:issue:`11501`, :pull:`#11627`). - Fix deadlocks when reading and writing netCDF files with dask at the same time. Combined locks now always acquire their locks in the same order, which previously depended on memory addresses, and a failed non-blocking acquire, diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 9ce39b2ba6a..90dfc5b78e0 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -538,6 +538,18 @@ def f(values, axis=None, skipna=None, **kwargs): nanname = "nan" + name func = getattr(nanops, nanname) + elif name in ["min", "max"] and dtypes.is_object(values.dtype): + # numpy's min/max of object arrays give order-dependent results + # when NaN is present, so compute ignoring nulls and then mask + # every slice that contains a null + from xarray.computation import nanops + + nanfunc = getattr(nanops, "nan" + name) + + def func(values, axis=None, **kwargs): + result = nanfunc(values, axis=axis, **kwargs) + return where_method(result, ~array_any(isnull(values), axis=axis)) + else: if name in ["sum", "prod"]: kwargs.pop("min_count", None) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 4bc073ab4b0..37eacad3ef0 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -292,6 +292,10 @@ def convert_non_numpy_type(data): and not isinstance(data.array, UNSUPPORTED_EXTENSION_ARRAY_TYPES) ): pandas_data = data.array + elif isinstance(data, pd.Series) and isinstance(data.dtype, pd.DatetimeTZDtype): + # Series.values is deprecated for timezone-aware data; convert + # to UTC datetime64 explicitly instead + pandas_data = data.to_numpy(dtype=f"datetime64[{data.dtype.unit}]") # type: ignore[assignment] else: pandas_data = data.values # type: ignore[assignment] if isinstance(pandas_data, NON_NUMPY_SUPPORTED_ARRAY_TYPES): diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index e94edeb1722..85ab2dc17bf 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -5877,6 +5877,12 @@ def test_roundtrip_string_encoded_characters(self) -> None: def test_roundtrip_coordinates_with_space(self) -> None: pass + @pytest.mark.skip( + reason="nulls are only replaced when encoding, which does not happen here" + ) + def test_roundtrip_stringdtype_nulls(self) -> None: + pass + def test_roundtrip_numpy_datetime_data(self) -> None: # Override method in DatasetIOBase - remove not applicable # save_kwargs diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 3d3d388287a..805b96b9160 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -5271,7 +5271,7 @@ def test_min( assert_identical(result1, expected1) result2 = ar.min(skipna=False) - if nanindex is not None and ar.dtype.kind != "O": + if nanindex is not None: expected2 = ar.isel(x=nanindex, drop=True) else: expected2 = expected1 @@ -5308,7 +5308,7 @@ def test_max( assert_identical(result1, expected1) result2 = ar.max(skipna=False) - if nanindex is not None and ar.dtype.kind != "O": + if nanindex is not None: expected2 = ar.isel(x=nanindex, drop=True) else: expected2 = expected1 @@ -5820,8 +5820,7 @@ def test_min( assert_identical(result2, expected0) # Default keeps attrs minindex = [ - x if y is None or ar.dtype.kind == "O" else y - for x, y in zip(minindex, nanindex, strict=True) + x if y is None else y for x, y in zip(minindex, nanindex, strict=True) ] expected2list = [ ar.isel(y=yi).isel(x=indi, drop=True) for yi, indi in enumerate(minindex) @@ -5870,8 +5869,7 @@ def test_max( assert_identical(result2, expected0) # Default keeps attrs maxindex = [ - x if y is None or ar.dtype.kind == "O" else y - for x, y in zip(maxindex, nanindex, strict=True) + x if y is None else y for x, y in zip(maxindex, nanindex, strict=True) ] expected2list = [ ar.isel(y=yi).isel(x=indi, drop=True) for yi, indi in enumerate(maxindex) diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 9a98ef0cbe7..ab0dd426499 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -578,6 +578,23 @@ def test_empty_axis_dtype(): assert_identical(ds.sum(dim="time")["var"], ds["var"]) +@pytest.mark.parametrize("func", ["min", "max"]) +@pytest.mark.parametrize( + "values", [[1.0, np.nan, 2.0], [np.nan, 1.0, 2.0], [True, np.nan, False]] +) +def test_min_max_object_propagates_nan(func, values) -> None: + da = DataArray(np.array(values, dtype=object)) + actual = getattr(da, func)(skipna=False) + assert np.isnan(actual.item()) + + da2d = DataArray(np.array([[3, np.nan], [2, 1]], dtype=object), dims=("x", "y")) + actual = getattr(da2d, func)("y", skipna=False) + expected = DataArray( + np.array([np.nan, 2 if func == "max" else 1], dtype=object), dims="x" + ) + assert_identical(actual, expected) + + def _reduce_params(): for dtype, use_dask, func, skipna in itertools.product( [float, int, np.float32, np.bool_], @@ -643,7 +660,9 @@ def test_reduce(dim_num, dtype, use_dask, func, skipna, aggdim): assert isinstance(da.data, dask_array_type) expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=5) assert_allclose(actual, expected, rtol=rtol) - else: + elif not (da.dtype.kind == "O" and not skipna and func in ["min", "max"]): + # pandas < 3.1 does not propagate NaN for object arrays here, + # see test_min_max_object_propagates_nan instead expected = series_reduce(da, func, skipna=skipna, dim=aggdim) assert_allclose(actual, expected, rtol=rtol) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 70a8ff5b027..22d737ba7af 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1870,10 +1870,12 @@ def test_colors(self) -> None: artist = self.darray.plot.contour( levels=[-0.5, 0.0, 0.5, 1.0], colors=["k", "r", "w", "b"] ) - assert artist.cmap.colors[:5] == ["k", "r", "w", "b"] # type: ignore[attr-defined,unused-ignore] + cmap = artist.cmap + assert isinstance(cmap, mpl.colors.ListedColormap) + assert cast(list[str], cmap.colors)[:5] == ["k", "r", "w", "b"] # the last color is now under "over" - assert self._color_as_tuple(artist.cmap.get_over()) == (0.0, 0.0, 1.0) + assert self._color_as_tuple(cmap.get_over()) == (0.0, 0.0, 1.0) def test_colors_np_levels(self) -> None: # https://github.com/pydata/xarray/issues/3284 @@ -1882,7 +1884,7 @@ def test_colors_np_levels(self) -> None: cmap = artist.cmap assert isinstance(cmap, mpl.colors.ListedColormap) - assert artist.cmap.colors[:5] == ["k", "r", "w", "b"] # type: ignore[attr-defined,unused-ignore] + assert cast(list[str], cmap.colors)[:5] == ["k", "r", "w", "b"] # the last color is now under "over" assert self._color_as_tuple(cmap.get_over()) == (0.0, 0.0, 1.0) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 1481fb5c548..45bfb832a83 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -2837,7 +2837,7 @@ def test_tz_datetime(self) -> None: warnings.simplefilter("ignore") actual2: T_DuckArray = as_compatible_data(series) - np.testing.assert_array_equal(actual2, np.asarray(series.values)) + np.testing.assert_array_equal(actual2, series.to_numpy(dtype="datetime64[s]")) assert actual2.dtype == np.dtype("datetime64[s]") def test_full_like(self) -> None: