[pull] main from pydata:main

.github/workflows/benchmarks-last-release.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       ASV_DIR: "./asv_bench"
-      CONDA_ENV_FILE: ci/requirements/environment.yml
+      CONDA_ENV_FILE: ci/requirements/environment-benchmark.yml
 
     steps:
       # We need the full repo to avoid this issue
@@ -29,11 +29,13 @@ jobs:
         with:
           micromamba-version: "1.5.10-0"
           environment-file: ${{env.CONDA_ENV_FILE}}
-          environment-name: xarray-tests
+          environment-name: xarray-benchmark
           cache-environment: true
           cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark"
+          # add "build" because of https://github.com/airspeed-velocity/asv/issues/1385
           create-args: >-
             asv
+            python-build
 
       - name: "Get Previous tag"
         id: previoustag
@@ -53,6 +55,8 @@ jobs:
           GITHUB_TAG: ${{ steps.previoustag.outputs.tag }}
         run: |
           set -x
+          # Fail if asv itself fails, not just when the log matches the grep below
+          set -o pipefail
           # ID this runner
           asv machine --yes
           echo "Baseline: $GITHUB_TAG"

ci/policy.yaml

@@ -27,6 +27,7 @@ policy:
     - pytz
     - pytest-reportlog
     - pyarrow # transitive dependency of dask.dataframe, not an xarray dependency
+    - google-crc32c # used by numcodecs if installed, not an xarray dependency
   # these packages don't fail the CI, but will be printed in the report
   ignored_violations:
     - array-api-strict

conftest.py

@@ -2,7 +2,9 @@
 
 import os
 
+import numpy as np
 import pytest
+from packaging.version import Version
 
 
 def _use_dask_array(config: pytest.Config) -> bool:
@@ -47,6 +49,13 @@ def pytest_addoption(parser: pytest.Parser):
 
 
 def pytest_configure(config: pytest.Config):
+    if Version(np.__version__) < Version("2"):
+        # raised by `numpy` < 2 for code in `h5netcdf` reading fill values. Not scoped
+        # to a module because warnings re-emitted by `pytest.warns` lose their module.
+        config.addinivalue_line(
+            "filterwarnings",
+            "ignore:Conversion of an array with ndim > 0 to a scalar:DeprecationWarning",
+        )
     config.addinivalue_line(
         "markers",
         "skip_with_dask_array: skip when dask-array is registered as xarray's dask chunk manager",
@@ -55,6 +64,11 @@ def pytest_configure(config: pytest.Config):
         "markers",
         "xfail_with_dask_array: xfail when dask-array is registered as xarray's dask chunk manager",
     )
+    config.addinivalue_line(
+        "markers",
+        "skip_if_param(*, reason, condition=True, **params): skip the test cases whose "
+        "parametrized arguments or fixtures equal all of the given params",
+    )
     if not _use_dask_array(config):
         return
 
@@ -64,22 +78,20 @@ def pytest_configure(config: pytest.Config):
 def pytest_runtest_setup(item):
     if _use_dask_array(item.config):
         _register_dask_array()
-    # based on https://stackoverflow.com/questions/47559524
-    if "flaky" in item.keywords and not item.config.getoption("--run-flaky"):
-        pytest.skip("set --run-flaky option to run flaky tests")
-    if "network" in item.keywords and not item.config.getoption("--run-network-tests"):
-        pytest.skip(
-            "set --run-network-tests to run test requiring an internet connection"
-        )
+    # The mypy yaml test items have no line number, which pytest requires to
+    # report a skip coming from a mark, so these have to be skipped at runtime.
     if any("mypy" in m.name for m in item.own_markers) and not item.config.getoption(
         "--run-mypy"
     ):
         pytest.skip("set --run-mypy option to run mypy tests")
 
 
 # See https://docs.pytest.org/en/stable/example/markers.html#automatically-adding-markers-based-on-test-names
-def pytest_collection_modifyitems(items):
+def pytest_collection_modifyitems(config: pytest.Config, items):
+    run_flaky = config.getoption("--run-flaky")
+    run_network = config.getoption("--run-network-tests")
     for item in items:
+        _apply_skip_if_param(item)
         if "mypy" in item.nodeid:
             # IMPORTANT: mypy type annotation tests leverage the pytest-mypy-plugins
             # plugin, and are thus written in test_*.yml files.  As such, there are
@@ -88,6 +100,17 @@ def pytest_collection_modifyitems(items):
             # marking approach, meaning that each test case must contain "mypy" in the
             # name.
             item.add_marker(pytest.mark.mypy)
+        # based on https://stackoverflow.com/questions/47559524
+        if "flaky" in item.keywords and not run_flaky:
+            item.add_marker(
+                pytest.mark.skip(reason="set --run-flaky option to run flaky tests")
+            )
+        if "network" in item.keywords and not run_network:
+            item.add_marker(
+                pytest.mark.skip(
+                    reason="set --run-network-tests to run test requiring an internet connection"
+                )
+            )
         if _use_dask_array(item.config) and "skip_with_dask_array" in item.keywords:
             item.add_marker(
                 pytest.mark.skip(reason="skipped with dask-array chunk manager")
@@ -102,6 +125,25 @@ def pytest_collection_modifyitems(items):
             item.add_marker(pytest.mark.xfail(**kwargs))
 
 
+def _apply_skip_if_param(item: pytest.Item) -> None:
+    markers = list(item.iter_markers("skip_if_param"))
+    if not markers:
+        return
+    callspec = getattr(item, "callspec", None)
+    params = callspec.params if callspec is not None else {}
+    for marker in markers:
+        kwargs = dict(marker.kwargs)
+        reason = kwargs.pop("reason")
+        condition = kwargs.pop("condition", True)
+        if missing := set(kwargs) - set(params):
+            raise pytest.UsageError(
+                f"{item.nodeid}: skip_if_param requires the test to be "
+                f"parametrized over {sorted(missing)}"
+            )
+        if condition and all(params[name] == value for name, value in kwargs.items()):
+            item.add_marker(pytest.mark.skip(reason=reason))
+
+
 @pytest.fixture(autouse=True)
 def add_standard_imports(doctest_namespace, tmpdir):
     import numpy as np

doc/whats-new.rst

@@ -53,6 +53,15 @@ Deprecations
 Bug Fixes
 ~~~~~~~~~
 
+- :py:meth:`DataArray.roll` and :py:meth:`Dataset.roll` now return unchanged
+  empty results when rolling an empty dimension, including with
+  ``roll_coords=True``, instead of raising ``ZeroDivisionError`` (:pull:`11613`).
+  By `Matthias Schabel <https://github.com/matthiasschabel>`_.
+- Fixed :py:meth:`~xarray.indexes.RangeIndex.arange` producing a negative-sized
+  index instead of an empty index when the interval direction conflicts with
+  the sign of ``step`` (:pull:`11623`).
+  By `Ahmet Kamer Çivi <https://github.com/lowgame>`_.
+
 - Fix async zarr tests using ``wraps`` with ``autospec=True`` on async methods,
   which caused ``AsyncMock`` objects to leak through instead of real array data
   (:pull:`11232`).
@@ -387,6 +396,10 @@ Bug Fixes
   By `Emmanuel Ferdman <https://github.com/emmanuel-ferdman>`_.
 - :func:`combine_by_coords` no longer returns an empty dataset when a generator is passed as ``data_objects`` (:issue:`10114`, :pull:`11265`).
   By `Amartya Anand <https://github.com/SurfyPenguin>`_.
+- Preserve non-grouped coordinates in fallback ``groupby`` reductions when grouping
+  by a non-leading dimension reorders the underlying variable dimensions
+  (:issue:`11188`, :pull:`11290`).
+  By `Sarthak <https://github.com/Sarthak160>`_.
 - Fix h5netcdf backend module detection and ros3 tests (:issue:`11243`, :pull:`11274`).
   By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
 

pixi.toml

@@ -142,6 +142,8 @@ sparse = "0.15.*"
 toolz = "1.0.*"
 zarr = "3.0.*"
 pyarrow = "*"        # required by dask.dataframe
+# numcodecs warns on import when crc32c is used instead of google-crc32c
+google-crc32c = "*"
 
 
 [feature.test.dependencies]

properties/test_encode_decode.py

@@ -49,15 +49,14 @@ def test_CFScaleOffset_coder_roundtrip(original) -> None:
     xr.testing.assert_identical(original, roundtripped)
 
 
-@given(
-    real=st.floats(allow_nan=True, allow_infinity=True),
-    imag=st.floats(allow_nan=True, allow_infinity=True),
-    dtype=st.sampled_from([np.complex64, np.complex128]),
-)
-def test_FillValueCoder_complex_roundtrip(real, imag, dtype) -> None:
+@given(data=st.data(), dtype=st.sampled_from([np.complex64, np.complex128]))
+def test_FillValueCoder_complex_roundtrip(data, dtype) -> None:
     from xarray.backends.zarr import FillValueCoder
 
-    value = dtype(complex(real, imag))
+    # draw values representable in `dtype` to avoid overflow when casting
+    value = dtype(
+        data.draw(npst.from_dtype(np.dtype(dtype), allow_nan=True, allow_infinity=True))
+    )
     encoded = FillValueCoder.encode(value, np.dtype(dtype))
     decoded = FillValueCoder.decode(encoded, np.dtype(dtype))
     np.testing.assert_equal(

properties/test_pandas_roundtrip.py

@@ -113,7 +113,6 @@ def test_roundtrip_pandas_series(ser, ix_name) -> None:
 )
 
 
-@pytest.mark.xfail
 @given(numeric_homogeneous_dataframe)
 def test_roundtrip_pandas_dataframe(df) -> None:
     # Need to name the indexes, otherwise Xarray names them 'dim_0', 'dim_1'.
@@ -122,7 +121,8 @@ def test_roundtrip_pandas_dataframe(df) -> None:
     arr = xr.DataArray(df)
     roundtripped = arr.to_pandas()
     pd.testing.assert_frame_equal(df, cast(pd.DataFrame, roundtripped))
-    xr.testing.assert_identical(arr, roundtripped.to_xarray())
+    # DataFrame.to_xarray returns a Dataset, so convert back to a DataArray
+    xr.testing.assert_identical(arr, xr.DataArray(roundtripped))
 
 
 @pytest.mark.xfail(reason="https://github.com/pandas-dev/pandas/issues/65712")

pyproject.toml

@@ -358,15 +358,20 @@ filterwarnings = [
   "error:::xarray.*",
   # Zarr 2 V3 implementation
   "default:Zarr-Python is not in alignment with the final V3 specification",
-  # TODO: this is raised for vlen-utf8, consolidated metadata, U1 dtype
-  "default:is currently not part .* the Zarr version 3 specification.",
+  # raised by zarr < 3.1 for vlen-utf8, consolidated metadata, U1 dtype
+  "ignore:.* is currently not part .* the Zarr (version|format) 3 specification.:UserWarning",
   # Zarr V3 data type specifications warnings - very repetitive
   "ignore:The data type .* does not have a Zarr V3 specification",
   "ignore:Consolidated metadata is currently not part",
-  # raised by `numpy` for code in `netcdf4-python`
-  "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.::xarray.backends.netCDF4_",
-  "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.::xarray.tests.test_backends",
-  "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.::xarray.tests.test_backends_datatree",
+  # raised by `numpy` for code in `netcdf4-python`. Not scoped to a module
+  # because warnings re-emitted by `pytest.warns` lose their module.
+  "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.:DeprecationWarning",
+  # raised by `pyarrow` for `pandas` ranking arrow-backed strings
+  "ignore:Specifying null_placement in RankOptions is deprecated:FutureWarning:pyarrow",
+  # raised by `numpy` for `np.datetime64("NaT")` in `flox`
+  "ignore:The 'generic' unit for NumPy timedelta is deprecated:DeprecationWarning:flox",
+  # raised by `numpy` for code in `netcdf4-python` < 1.7 when indexing with empty arrays
+  "ignore:The truth value of an empty array is ambiguous:DeprecationWarning:netCDF4",
   # TODO: remove once we know how to deal with a changed signature in protocols
   "default:::xarray.tests.test_strategies",
 ]

xarray/computation/nanops.py

@@ -153,11 +153,21 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0):
     if a.dtype.kind == "O":
         return _nanvar_object(a, axis=axis, dtype=dtype, ddof=ddof)
 
-    return nputils.nanvar(a, axis=axis, dtype=dtype, ddof=ddof)
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore", r"Degrees of freedom <= 0 for slice", category=RuntimeWarning
+        )
+
+        return nputils.nanvar(a, axis=axis, dtype=dtype, ddof=ddof)
 
 
 def nanstd(a, axis=None, dtype=None, out=None, ddof=0):
-    return nputils.nanstd(a, axis=axis, dtype=dtype, ddof=ddof)
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore", r"Degrees of freedom <= 0 for slice", category=RuntimeWarning
+        )
+
+        return nputils.nanstd(a, axis=axis, dtype=dtype, ddof=ddof)
 
 
 def nanprod(a, axis=None, dtype=None, out=None, min_count=None):

xarray/core/dataarray.py

@@ -634,7 +634,7 @@ def _replace_maybe_drop_dims(
             indexes = self._indexes
         elif set(self.dims) == set(variable.dims):
             # Shape has changed (e.g. from reduce(..., keepdims=True)
-            new_sizes = dict(zip(self.dims, variable.shape, strict=True))
+            new_sizes = dict(variable.sizes)
             coords = {
                 k: v
                 for k, v in self._coords.items()

xarray/core/extension_array.py

@@ -111,14 +111,17 @@ def as_extension_array(
 
 @implements(np.result_type)
 def __extension_duck_array__result_type(
-    *arrays_and_dtypes: list[
-        np.typing.ArrayLike | np.typing.DTypeLike | ExtensionDtype | ExtensionArray
-    ],
+    *arrays_and_dtypes: np.typing.ArrayLike
+    | np.typing.DTypeLike
+    | ExtensionDtype
+    | ExtensionArray,
 ) -> DtypeObj:
     extension_arrays_and_dtypes: list[ExtensionDtype | ExtensionArray] = [
         cast(ExtensionDtype | ExtensionArray, x)
         for x in arrays_and_dtypes
-        if is_allowed_extension_array(x) or is_allowed_extension_array_dtype(x)
+        if is_allowed_extension_array(x)
+        # strings are scalar values here; pandas would interpret them as dtype names
+        or (not isinstance(x, str | bytes) and is_allowed_extension_array_dtype(x))
     ]
     if not extension_arrays_and_dtypes:
         return NotImplemented
@@ -136,7 +139,7 @@ def __extension_duck_array__result_type(
     other_stuff = [
         x
         for x in arrays_and_dtypes
-        if not is_allowed_extension_array_dtype(x) and not is_scalar(x)
+        if not is_scalar(x) and not is_allowed_extension_array_dtype(x)
     ]
     # We implement one special case: when possible, preserve Categoricals (avoid promoting
     # to object) by merging the categories of all given Categoricals + scalars + NA.

xarray/core/indexes.py

@@ -934,7 +934,7 @@ def reindex_like(
         return {self.dim: get_indexer_nd(self.index, other.index, method, tolerance)}
 
     def roll(self, shifts: Mapping[Any, int]) -> PandasIndex:
-        shift = shifts[self.dim] % self.index.shape[0]
+        shift = shifts[self.dim] % (self.index.shape[0] or 1)
 
         if shift != 0:
             new_pd_idx = self.index[-shift:].append(self.index[:-shift])

xarray/core/variable.py

@@ -1356,7 +1356,7 @@ def pad(
     def _roll_one_dim(self, dim, count):
         axis = self.get_axis_num(dim)
 
-        count %= self.shape[axis]
+        count %= self.shape[axis] or 1
         if count != 0:
             indices = [slice(-count, None), slice(None, -count)]
         else:

xarray/indexes/range_index.py

@@ -279,7 +279,7 @@ def arange(
         if coord_name is None:
             coord_name = dim
 
-        size = math.ceil((stop - start) / step)
+        size = max(0, math.ceil((stop - start) / step))
 
         # Snap ``stop`` to ``start + size * step`` and keep the exact ``step`` so
         # that the materialized values match ``numpy.arange`` even when ``step``

xarray/tests/__init__.py

@@ -107,7 +107,7 @@ def get_dask_chunkmanager():
     warnings.filterwarnings(
         "ignore",
         message="'cgi' is deprecated and slated for removal in Python 3.13",
-        category=FutureWarning,
+        category=DeprecationWarning,
     )
     has_pydap, requires_pydap = _importorskip("pydap.client")
 has_netCDF4, requires_netCDF4 = _importorskip("netCDF4")
@@ -152,7 +152,12 @@ def get_dask_chunkmanager():
 
 has_bottleneck, requires_bottleneck = _importorskip("bottleneck")
 has_rasterio, requires_rasterio = _importorskip("rasterio")
-has_zarr, requires_zarr = _importorskip("zarr")
+with warnings.catch_warnings():
+    # raised when importing numcodecs with crc32c installed
+    warnings.filterwarnings(
+        "ignore", message="crc32c usage is deprecated", category=DeprecationWarning
+    )
+    has_zarr, requires_zarr = _importorskip("zarr")
 # Since min zarr is now >=3.0, these are aliases for has_zarr/requires_zarr.
 # Kept to avoid a large diff across test files.
 has_zarr_v3 = has_zarr
@@ -217,6 +222,8 @@ def get_dask_chunkmanager():
 has_flox_0_9_12, requires_flox_0_9_12 = _importorskip("flox", "0.9.12")
 
 has_array_api_strict, requires_array_api_strict = _importorskip("array_api_strict")
+has_jax, requires_jax = _importorskip("jax")
+has_aiobotocore, requires_aiobotocore = _importorskip("aiobotocore")
 
 parametrize_zarr_format = pytest.mark.parametrize(
     "zarr_format",