diff --git a/.github/workflows/benchmarks-last-release.yml b/.github/workflows/benchmarks-last-release.yml index e6f89c3e656..f64506c48eb 100644 --- a/.github/workflows/benchmarks-last-release.yml +++ b/.github/workflows/benchmarks-last-release.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest env: ASV_DIR: "./asv_bench" - CONDA_ENV_FILE: ci/requirements/environment.yml + CONDA_ENV_FILE: ci/requirements/environment-benchmark.yml steps: # We need the full repo to avoid this issue @@ -29,11 +29,13 @@ jobs: with: micromamba-version: "1.5.10-0" environment-file: ${{env.CONDA_ENV_FILE}} - environment-name: xarray-tests + environment-name: xarray-benchmark cache-environment: true cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" + # add "build" because of https://github.com/airspeed-velocity/asv/issues/1385 create-args: >- asv + python-build - name: "Get Previous tag" id: previoustag @@ -53,6 +55,8 @@ jobs: GITHUB_TAG: ${{ steps.previoustag.outputs.tag }} run: | set -x + # Fail if asv itself fails, not just when the log matches the grep below + set -o pipefail # ID this runner asv machine --yes echo "Baseline: $GITHUB_TAG" diff --git a/ci/policy.yaml b/ci/policy.yaml index 3b9960087a4..b9b1a42da35 100644 --- a/ci/policy.yaml +++ b/ci/policy.yaml @@ -27,6 +27,7 @@ policy: - pytz - pytest-reportlog - pyarrow # transitive dependency of dask.dataframe, not an xarray dependency + - google-crc32c # used by numcodecs if installed, not an xarray dependency # these packages don't fail the CI, but will be printed in the report ignored_violations: - array-api-strict diff --git a/conftest.py b/conftest.py index 825df8f2985..005f0168dc6 100644 --- a/conftest.py +++ b/conftest.py @@ -2,7 +2,9 @@ import os +import numpy as np import pytest +from packaging.version import Version def _use_dask_array(config: pytest.Config) -> bool: @@ -47,6 +49,13 @@ def pytest_addoption(parser: pytest.Parser): def pytest_configure(config: pytest.Config): + if Version(np.__version__) < Version("2"): + # raised by `numpy` < 2 for code in `h5netcdf` reading fill values. Not scoped + # to a module because warnings re-emitted by `pytest.warns` lose their module. + config.addinivalue_line( + "filterwarnings", + "ignore:Conversion of an array with ndim > 0 to a scalar:DeprecationWarning", + ) config.addinivalue_line( "markers", "skip_with_dask_array: skip when dask-array is registered as xarray's dask chunk manager", @@ -55,6 +64,11 @@ def pytest_configure(config: pytest.Config): "markers", "xfail_with_dask_array: xfail when dask-array is registered as xarray's dask chunk manager", ) + config.addinivalue_line( + "markers", + "skip_if_param(*, reason, condition=True, **params): skip the test cases whose " + "parametrized arguments or fixtures equal all of the given params", + ) if not _use_dask_array(config): return @@ -64,13 +78,8 @@ def pytest_configure(config: pytest.Config): def pytest_runtest_setup(item): if _use_dask_array(item.config): _register_dask_array() - # based on https://stackoverflow.com/questions/47559524 - if "flaky" in item.keywords and not item.config.getoption("--run-flaky"): - pytest.skip("set --run-flaky option to run flaky tests") - if "network" in item.keywords and not item.config.getoption("--run-network-tests"): - pytest.skip( - "set --run-network-tests to run test requiring an internet connection" - ) + # The mypy yaml test items have no line number, which pytest requires to + # report a skip coming from a mark, so these have to be skipped at runtime. if any("mypy" in m.name for m in item.own_markers) and not item.config.getoption( "--run-mypy" ): @@ -78,8 +87,11 @@ def pytest_runtest_setup(item): # See https://docs.pytest.org/en/stable/example/markers.html#automatically-adding-markers-based-on-test-names -def pytest_collection_modifyitems(items): +def pytest_collection_modifyitems(config: pytest.Config, items): + run_flaky = config.getoption("--run-flaky") + run_network = config.getoption("--run-network-tests") for item in items: + _apply_skip_if_param(item) if "mypy" in item.nodeid: # IMPORTANT: mypy type annotation tests leverage the pytest-mypy-plugins # plugin, and are thus written in test_*.yml files. As such, there are @@ -88,6 +100,17 @@ def pytest_collection_modifyitems(items): # marking approach, meaning that each test case must contain "mypy" in the # name. item.add_marker(pytest.mark.mypy) + # based on https://stackoverflow.com/questions/47559524 + if "flaky" in item.keywords and not run_flaky: + item.add_marker( + pytest.mark.skip(reason="set --run-flaky option to run flaky tests") + ) + if "network" in item.keywords and not run_network: + item.add_marker( + pytest.mark.skip( + reason="set --run-network-tests to run test requiring an internet connection" + ) + ) if _use_dask_array(item.config) and "skip_with_dask_array" in item.keywords: item.add_marker( pytest.mark.skip(reason="skipped with dask-array chunk manager") @@ -102,6 +125,25 @@ def pytest_collection_modifyitems(items): item.add_marker(pytest.mark.xfail(**kwargs)) +def _apply_skip_if_param(item: pytest.Item) -> None: + markers = list(item.iter_markers("skip_if_param")) + if not markers: + return + callspec = getattr(item, "callspec", None) + params = callspec.params if callspec is not None else {} + for marker in markers: + kwargs = dict(marker.kwargs) + reason = kwargs.pop("reason") + condition = kwargs.pop("condition", True) + if missing := set(kwargs) - set(params): + raise pytest.UsageError( + f"{item.nodeid}: skip_if_param requires the test to be " + f"parametrized over {sorted(missing)}" + ) + if condition and all(params[name] == value for name, value in kwargs.items()): + item.add_marker(pytest.mark.skip(reason=reason)) + + @pytest.fixture(autouse=True) def add_standard_imports(doctest_namespace, tmpdir): import numpy as np diff --git a/doc/whats-new.rst b/doc/whats-new.rst index da7259da069..bfbff532c17 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -53,6 +53,15 @@ Deprecations Bug Fixes ~~~~~~~~~ +- :py:meth:`DataArray.roll` and :py:meth:`Dataset.roll` now return unchanged + empty results when rolling an empty dimension, including with + ``roll_coords=True``, instead of raising ``ZeroDivisionError`` (:pull:`11613`). + By `Matthias Schabel `_. +- Fixed :py:meth:`~xarray.indexes.RangeIndex.arange` producing a negative-sized + index instead of an empty index when the interval direction conflicts with + the sign of ``step`` (:pull:`11623`). + By `Ahmet Kamer Çivi `_. + - Fix async zarr tests using ``wraps`` with ``autospec=True`` on async methods, which caused ``AsyncMock`` objects to leak through instead of real array data (:pull:`11232`). @@ -387,6 +396,10 @@ Bug Fixes By `Emmanuel Ferdman `_. - :func:`combine_by_coords` no longer returns an empty dataset when a generator is passed as ``data_objects`` (:issue:`10114`, :pull:`11265`). By `Amartya Anand `_. +- Preserve non-grouped coordinates in fallback ``groupby`` reductions when grouping + by a non-leading dimension reorders the underlying variable dimensions + (:issue:`11188`, :pull:`11290`). + By `Sarthak `_. - Fix h5netcdf backend module detection and ros3 tests (:issue:`11243`, :pull:`11274`). By `Kai Mühlbauer `_. diff --git a/pixi.toml b/pixi.toml index 8768fa44978..4b097032331 100644 --- a/pixi.toml +++ b/pixi.toml @@ -142,6 +142,8 @@ sparse = "0.15.*" toolz = "1.0.*" zarr = "3.0.*" pyarrow = "*" # required by dask.dataframe +# numcodecs warns on import when crc32c is used instead of google-crc32c +google-crc32c = "*" [feature.test.dependencies] diff --git a/properties/test_encode_decode.py b/properties/test_encode_decode.py index bce2fc215c6..4443164a2e2 100644 --- a/properties/test_encode_decode.py +++ b/properties/test_encode_decode.py @@ -49,15 +49,14 @@ def test_CFScaleOffset_coder_roundtrip(original) -> None: xr.testing.assert_identical(original, roundtripped) -@given( - real=st.floats(allow_nan=True, allow_infinity=True), - imag=st.floats(allow_nan=True, allow_infinity=True), - dtype=st.sampled_from([np.complex64, np.complex128]), -) -def test_FillValueCoder_complex_roundtrip(real, imag, dtype) -> None: +@given(data=st.data(), dtype=st.sampled_from([np.complex64, np.complex128])) +def test_FillValueCoder_complex_roundtrip(data, dtype) -> None: from xarray.backends.zarr import FillValueCoder - value = dtype(complex(real, imag)) + # draw values representable in `dtype` to avoid overflow when casting + value = dtype( + data.draw(npst.from_dtype(np.dtype(dtype), allow_nan=True, allow_infinity=True)) + ) encoded = FillValueCoder.encode(value, np.dtype(dtype)) decoded = FillValueCoder.decode(encoded, np.dtype(dtype)) np.testing.assert_equal( diff --git a/properties/test_pandas_roundtrip.py b/properties/test_pandas_roundtrip.py index 3098f17e455..543e0f81fb7 100644 --- a/properties/test_pandas_roundtrip.py +++ b/properties/test_pandas_roundtrip.py @@ -113,7 +113,6 @@ def test_roundtrip_pandas_series(ser, ix_name) -> None: ) -@pytest.mark.xfail @given(numeric_homogeneous_dataframe) def test_roundtrip_pandas_dataframe(df) -> None: # Need to name the indexes, otherwise Xarray names them 'dim_0', 'dim_1'. @@ -122,7 +121,8 @@ def test_roundtrip_pandas_dataframe(df) -> None: arr = xr.DataArray(df) roundtripped = arr.to_pandas() pd.testing.assert_frame_equal(df, cast(pd.DataFrame, roundtripped)) - xr.testing.assert_identical(arr, roundtripped.to_xarray()) + # DataFrame.to_xarray returns a Dataset, so convert back to a DataArray + xr.testing.assert_identical(arr, xr.DataArray(roundtripped)) @pytest.mark.xfail(reason="https://github.com/pandas-dev/pandas/issues/65712") diff --git a/pyproject.toml b/pyproject.toml index 8ce4391adae..6a011031765 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -358,15 +358,20 @@ filterwarnings = [ "error:::xarray.*", # Zarr 2 V3 implementation "default:Zarr-Python is not in alignment with the final V3 specification", - # TODO: this is raised for vlen-utf8, consolidated metadata, U1 dtype - "default:is currently not part .* the Zarr version 3 specification.", + # raised by zarr < 3.1 for vlen-utf8, consolidated metadata, U1 dtype + "ignore:.* is currently not part .* the Zarr (version|format) 3 specification.:UserWarning", # Zarr V3 data type specifications warnings - very repetitive "ignore:The data type .* does not have a Zarr V3 specification", "ignore:Consolidated metadata is currently not part", - # raised by `numpy` for code in `netcdf4-python` - "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.::xarray.backends.netCDF4_", - "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.::xarray.tests.test_backends", - "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.::xarray.tests.test_backends_datatree", + # raised by `numpy` for code in `netcdf4-python`. Not scoped to a module + # because warnings re-emitted by `pytest.warns` lose their module. + "ignore:Setting the shape on a NumPy array has been deprecated in NumPy 2.5.:DeprecationWarning", + # raised by `pyarrow` for `pandas` ranking arrow-backed strings + "ignore:Specifying null_placement in RankOptions is deprecated:FutureWarning:pyarrow", + # raised by `numpy` for `np.datetime64("NaT")` in `flox` + "ignore:The 'generic' unit for NumPy timedelta is deprecated:DeprecationWarning:flox", + # raised by `numpy` for code in `netcdf4-python` < 1.7 when indexing with empty arrays + "ignore:The truth value of an empty array is ambiguous:DeprecationWarning:netCDF4", # TODO: remove once we know how to deal with a changed signature in protocols "default:::xarray.tests.test_strategies", ] diff --git a/xarray/computation/nanops.py b/xarray/computation/nanops.py index c5a9d9bcc04..bd06b91b1bc 100644 --- a/xarray/computation/nanops.py +++ b/xarray/computation/nanops.py @@ -153,11 +153,21 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0): if a.dtype.kind == "O": return _nanvar_object(a, axis=axis, dtype=dtype, ddof=ddof) - return nputils.nanvar(a, axis=axis, dtype=dtype, ddof=ddof) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", r"Degrees of freedom <= 0 for slice", category=RuntimeWarning + ) + + return nputils.nanvar(a, axis=axis, dtype=dtype, ddof=ddof) def nanstd(a, axis=None, dtype=None, out=None, ddof=0): - return nputils.nanstd(a, axis=axis, dtype=dtype, ddof=ddof) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", r"Degrees of freedom <= 0 for slice", category=RuntimeWarning + ) + + return nputils.nanstd(a, axis=axis, dtype=dtype, ddof=ddof) def nanprod(a, axis=None, dtype=None, out=None, min_count=None): diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 5c576ff9680..8d201ae1a21 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -634,7 +634,7 @@ def _replace_maybe_drop_dims( indexes = self._indexes elif set(self.dims) == set(variable.dims): # Shape has changed (e.g. from reduce(..., keepdims=True) - new_sizes = dict(zip(self.dims, variable.shape, strict=True)) + new_sizes = dict(variable.sizes) coords = { k: v for k, v in self._coords.items() diff --git a/xarray/core/extension_array.py b/xarray/core/extension_array.py index d078d61e58a..e3ca1bbe788 100644 --- a/xarray/core/extension_array.py +++ b/xarray/core/extension_array.py @@ -111,14 +111,17 @@ def as_extension_array( @implements(np.result_type) def __extension_duck_array__result_type( - *arrays_and_dtypes: list[ - np.typing.ArrayLike | np.typing.DTypeLike | ExtensionDtype | ExtensionArray - ], + *arrays_and_dtypes: np.typing.ArrayLike + | np.typing.DTypeLike + | ExtensionDtype + | ExtensionArray, ) -> DtypeObj: extension_arrays_and_dtypes: list[ExtensionDtype | ExtensionArray] = [ cast(ExtensionDtype | ExtensionArray, x) for x in arrays_and_dtypes - if is_allowed_extension_array(x) or is_allowed_extension_array_dtype(x) + if is_allowed_extension_array(x) + # strings are scalar values here; pandas would interpret them as dtype names + or (not isinstance(x, str | bytes) and is_allowed_extension_array_dtype(x)) ] if not extension_arrays_and_dtypes: return NotImplemented @@ -136,7 +139,7 @@ def __extension_duck_array__result_type( other_stuff = [ x for x in arrays_and_dtypes - if not is_allowed_extension_array_dtype(x) and not is_scalar(x) + if not is_scalar(x) and not is_allowed_extension_array_dtype(x) ] # We implement one special case: when possible, preserve Categoricals (avoid promoting # to object) by merging the categories of all given Categoricals + scalars + NA. diff --git a/xarray/core/indexes.py b/xarray/core/indexes.py index ab9b7962020..e17b524afbf 100644 --- a/xarray/core/indexes.py +++ b/xarray/core/indexes.py @@ -934,7 +934,7 @@ def reindex_like( return {self.dim: get_indexer_nd(self.index, other.index, method, tolerance)} def roll(self, shifts: Mapping[Any, int]) -> PandasIndex: - shift = shifts[self.dim] % self.index.shape[0] + shift = shifts[self.dim] % (self.index.shape[0] or 1) if shift != 0: new_pd_idx = self.index[-shift:].append(self.index[:-shift]) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 0ece498c8ce..4bc073ab4b0 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1356,7 +1356,7 @@ def pad( def _roll_one_dim(self, dim, count): axis = self.get_axis_num(dim) - count %= self.shape[axis] + count %= self.shape[axis] or 1 if count != 0: indices = [slice(-count, None), slice(None, -count)] else: diff --git a/xarray/indexes/range_index.py b/xarray/indexes/range_index.py index d1e272b39c8..eecdd658e80 100644 --- a/xarray/indexes/range_index.py +++ b/xarray/indexes/range_index.py @@ -279,7 +279,7 @@ def arange( if coord_name is None: coord_name = dim - size = math.ceil((stop - start) / step) + size = max(0, math.ceil((stop - start) / step)) # Snap ``stop`` to ``start + size * step`` and keep the exact ``step`` so # that the materialized values match ``numpy.arange`` even when ``step`` diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 15326b1d7e7..f50e662af64 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -107,7 +107,7 @@ def get_dask_chunkmanager(): warnings.filterwarnings( "ignore", message="'cgi' is deprecated and slated for removal in Python 3.13", - category=FutureWarning, + category=DeprecationWarning, ) has_pydap, requires_pydap = _importorskip("pydap.client") has_netCDF4, requires_netCDF4 = _importorskip("netCDF4") @@ -152,7 +152,12 @@ def get_dask_chunkmanager(): has_bottleneck, requires_bottleneck = _importorskip("bottleneck") has_rasterio, requires_rasterio = _importorskip("rasterio") -has_zarr, requires_zarr = _importorskip("zarr") +with warnings.catch_warnings(): + # raised when importing numcodecs with crc32c installed + warnings.filterwarnings( + "ignore", message="crc32c usage is deprecated", category=DeprecationWarning + ) + has_zarr, requires_zarr = _importorskip("zarr") # Since min zarr is now >=3.0, these are aliases for has_zarr/requires_zarr. # Kept to avoid a large diff across test files. has_zarr_v3 = has_zarr @@ -217,6 +222,8 @@ def get_dask_chunkmanager(): has_flox_0_9_12, requires_flox_0_9_12 = _importorskip("flox", "0.9.12") has_array_api_strict, requires_array_api_strict = _importorskip("array_api_strict") +has_jax, requires_jax = _importorskip("jax") +has_aiobotocore, requires_aiobotocore = _importorskip("aiobotocore") parametrize_zarr_format = pytest.mark.parametrize( "zarr_format", diff --git a/xarray/tests/conftest.py b/xarray/tests/conftest.py index b8955b9e63d..141b8abcfd6 100644 --- a/xarray/tests/conftest.py +++ b/xarray/tests/conftest.py @@ -8,7 +8,7 @@ import xarray as xr from xarray import DataArray, Dataset, DataTree -from xarray.tests import create_test_data, has_cftime, requires_dask +from xarray.tests import create_test_data, requires_cftime, requires_dask @pytest.fixture(autouse=True) @@ -32,8 +32,13 @@ def handle_numpy_1_warnings(): yield -@pytest.fixture(params=["numpy", pytest.param("dask", marks=requires_dask)]) -def backend(request): +@pytest.fixture( + params=[ + pytest.param(False, id="numpy"), + pytest.param(True, id="dask", marks=requires_dask), + ] +) +def use_dask(request): return request.param @@ -52,10 +57,9 @@ def compute_backend(request): yield request.param -@pytest.fixture(params=[1]) -def ds(request, backend): - if request.param == 1: - ds = Dataset( +def _create_test_ds(variant) -> Dataset: + if variant == 1: + return Dataset( dict( z1=(["y", "x"], np.random.randn(2, 8)), z2=(["time", "y"], np.random.randn(10, 2)), @@ -67,8 +71,8 @@ def ds(request, backend): y=range(2), ), ) - elif request.param == 2: - ds = Dataset( + elif variant == 2: + return Dataset( dict( z1=(["time", "y"], np.random.randn(10, 2)), z2=(["time"], np.random.randn(10)), @@ -81,51 +85,60 @@ def ds(request, backend): y=range(2), ), ) - elif request.param == 3: - ds = create_test_data() - else: - raise ValueError - - if backend == "dask": - return ds.chunk() + elif variant == 3: + return create_test_data() + raise ValueError(f"unknown ds variant {variant!r}") - return ds - -@pytest.fixture(params=[1]) -def da(request, backend): - if request.param == 1: +def _create_test_da(variant) -> DataArray: + if variant == 1: times = pd.date_range("2000-01-01", freq="1D", periods=21) - da = DataArray( + return DataArray( np.random.random((3, 21, 4)), dims=("a", "time", "x"), coords=dict(time=times), ) - - if request.param == 2: - da = DataArray([0, np.nan, 1, 2, np.nan, 3, 4, 5, np.nan, 6, 7], dims="time") - - if request.param == "repeating_ints": - da = DataArray( + elif variant == 2: + return DataArray([0, np.nan, 1, 2, np.nan, 3, 4, 5, np.nan, 6, 7], dims="time") + elif variant == "repeating_ints": + return DataArray( np.tile(np.arange(12), 5).reshape(5, 4, 3), coords={"x": list("abc"), "y": list("defg")}, dims=list("zyx"), ) + raise ValueError(f"unknown da variant {variant!r}") - if backend == "dask": - return da.chunk() - elif backend == "numpy": - return da - else: - raise ValueError + +@pytest.fixture(params=[1]) +def ds(request, use_dask) -> Dataset: + """Test Dataset, run once with numpy and once with dask.""" + ds = _create_test_ds(request.param) + return ds.chunk() if use_dask else ds + + +@pytest.fixture(params=[1]) +def ds_numpy(request) -> Dataset: + """Test Dataset backed by numpy only.""" + return _create_test_ds(request.param) + + +@pytest.fixture(params=[1]) +def da(request, use_dask) -> DataArray: + """Test DataArray, run once with numpy and once with dask.""" + da = _create_test_da(request.param) + return da.chunk() if use_dask else da + + +@pytest.fixture(params=[1]) +def da_numpy(request) -> DataArray: + """Test DataArray backed by numpy only.""" + return _create_test_da(request.param) @pytest.fixture( params=[ - False, - pytest.param( - True, marks=pytest.mark.skipif(not has_cftime, reason="no cftime") - ), + pytest.param(False, id="datetime64"), + pytest.param(True, id="cftime", marks=requires_cftime), ] ) def use_cftime(request): @@ -138,7 +151,7 @@ def type(request): @pytest.fixture(params=[1]) -def d(request, backend, type) -> DataArray | Dataset: +def d(request, use_dask, type) -> DataArray | Dataset: """ For tests which can test either a DataArray or a Dataset. """ @@ -165,32 +178,9 @@ def d(request, backend, type) -> DataArray | Dataset: else: raise ValueError - if backend == "dask": + if use_dask: return result.chunk() - elif backend == "numpy": - return result - else: - raise ValueError - - -@pytest.fixture -def byte_attrs_dataset(): - """For testing issue #9407""" - null_byte = b"\x00" - other_bytes = bytes(range(1, 256)) - ds = Dataset({"x": 1}, coords={"x_coord": [1]}) - ds["x"].attrs["null_byte"] = null_byte - ds["x"].attrs["other_bytes"] = other_bytes - - expected = ds.copy() - expected["x"].attrs["null_byte"] = "" - expected["x"].attrs["other_bytes"] = other_bytes.decode(errors="replace") - - return { - "input": ds, - "expected": expected, - "h5netcdf_error": r"Invalid value provided for attribute .*: .*\. Null characters .*", - } + return result @pytest.fixture(scope="module") diff --git a/xarray/tests/test_accessor_dt.py b/xarray/tests/test_accessor_dt.py index c3e36724bee..093236a5b26 100644 --- a/xarray/tests/test_accessor_dt.py +++ b/xarray/tests/test_accessor_dt.py @@ -580,8 +580,6 @@ def cftime_rounding_dataarray(cftime_date_type): @requires_cftime -@requires_dask -@pytest.mark.parametrize("use_dask", [False, True]) def test_cftime_floor_accessor( cftime_rounding_dataarray, cftime_date_type, use_dask ) -> None: @@ -611,8 +609,6 @@ def test_cftime_floor_accessor( @requires_cftime -@requires_dask -@pytest.mark.parametrize("use_dask", [False, True]) def test_cftime_ceil_accessor( cftime_rounding_dataarray, cftime_date_type, use_dask ) -> None: @@ -642,8 +638,6 @@ def test_cftime_ceil_accessor( @requires_cftime -@requires_dask -@pytest.mark.parametrize("use_dask", [False, True]) def test_cftime_round_accessor( cftime_rounding_dataarray, cftime_date_type, use_dask ) -> None: @@ -672,16 +666,6 @@ def test_cftime_round_accessor( assert_identical(result, expected) -@pytest.mark.parametrize( - "use_cftime", - [False, pytest.param(True, marks=requires_cftime)], - ids=lambda x: f"use_cftime={x}", -) -@pytest.mark.parametrize( - "use_dask", - [False, pytest.param(True, marks=requires_dask)], - ids=lambda x: f"use_dask={x}", -) def test_decimal_year(use_cftime, use_dask) -> None: year = 2000 periods = 10 diff --git a/xarray/tests/test_assertions.py b/xarray/tests/test_assertions.py index a78a23bb057..94606a3b9e9 100644 --- a/xarray/tests/test_assertions.py +++ b/xarray/tests/test_assertions.py @@ -8,7 +8,7 @@ import xarray as xr from xarray.core.coordinates import Coordinates from xarray.core.indexes import Index -from xarray.tests import has_dask +from xarray.tests import requires_dask, requires_pint try: from dask.array import from_array as dask_from_array @@ -23,14 +23,11 @@ def quantity(x): return unit_registry.Quantity(x, "m") - has_pint = True except ImportError: def quantity(x): return x - has_pint = False - def test_allclose_regression() -> None: x = xr.DataArray(1.01) @@ -144,12 +141,12 @@ def test_assert_equal_transpose_datatree() -> None: pytest.param( dask_from_array, id="dask", - marks=pytest.mark.skipif(not has_dask, reason="requires dask"), + marks=requires_dask, ), pytest.param( quantity, id="pint", - marks=pytest.mark.skipif(not has_pint, reason="requires pint"), + marks=requires_pint, ), ), ) @@ -180,12 +177,12 @@ def test_assert_duckarray_equal_failing(duckarray, obj1, obj2) -> None: pytest.param( dask_from_array, id="dask", - marks=pytest.mark.skipif(not has_dask, reason="requires dask"), + marks=requires_dask, ), pytest.param( quantity, id="pint", - marks=pytest.mark.skipif(not has_pint, reason="requires pint"), + marks=requires_pint, ), ), ) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a56d024bdb6..c52c4674ea7 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -86,6 +86,7 @@ network, parametrize_zarr_format, raise_if_dask_computes, + requires_aiobotocore, requires_cftime, requires_dask, requires_fsspec, @@ -102,6 +103,7 @@ requires_scipy_or_netCDF4, requires_zarr, requires_zarr_v3, + requires_zarr_v3_async_oindex, ) from xarray.tests.test_coding_times import ( _ALL_CALENDARS, @@ -143,23 +145,44 @@ from xarray.backends.api import T_NetcdfEngine, T_NetcdfTypes -@pytest.fixture(scope="module", params=ZARR_FORMATS) +@pytest.fixture(params=ZARR_FORMATS) def default_zarr_format(request) -> Generator[None, None]: with zarr.config.set(default_zarr_format=request.param): yield -def skip_if_zarr_format_3(reason: str): - if has_zarr and zarr.config["default_zarr_format"] == 3: - pytest.skip(reason=f"Unsupported with zarr_format=3: {reason}") +# These marks require the test to use the ``default_zarr_format`` fixture. +def skip_if_zarr_format_3(reason: str, condition: bool = True) -> pytest.MarkDecorator: + return pytest.mark.skip_if_param( + default_zarr_format=3, + reason=f"Unsupported with zarr_format=3: {reason}", + condition=condition, + ) + + +def skip_if_zarr_format_2(reason: str, condition: bool = True) -> pytest.MarkDecorator: + return pytest.mark.skip_if_param( + default_zarr_format=2, + reason=f"Unsupported with zarr_format=2: {reason}", + condition=condition, + ) -def skip_if_zarr_format_2(reason: str): - if has_zarr and zarr.config["default_zarr_format"] == 2: - pytest.skip(reason=f"Unsupported with zarr_format=2: {reason}") +# Requires the test to use the ``tmp_store`` fixture. +skip_if_zip_store = pytest.mark.skip_if_param( + tmp_store="ZipStore", + reason="zarr-python 3.x doesn't support reopening ZipStore with a new mode.", +) ON_WINDOWS = sys.platform == "win32" + +if has_netCDF4: + NETCDFC_VERSION: Version | None = Version( + nc4.getlibversion().split()[0].split("-development")[0] + ) +else: + NETCDFC_VERSION = None default_value = object() @@ -377,6 +400,28 @@ def create_boolean_data() -> Dataset: ) +MASK_AND_SCALE_DATA = [ + ( + create_unsigned_masked_scaled_data, + create_encoded_unsigned_masked_scaled_data, + ), + pytest.param( + create_bad_unsigned_masked_scaled_data, + create_bad_encoded_unsigned_masked_scaled_data, + marks=pytest.mark.xfail(reason="Bad _Unsigned attribute."), + ), + ( + create_signed_masked_scaled_data, + create_encoded_signed_masked_scaled_data, + ), + ( + create_unsigned_false_masked_scaled_data, + create_encoded_unsigned_false_masked_scaled_data, + ), + (create_masked_and_scaled_data, create_encoded_masked_and_scaled_data), +] + + class TestCommon: def test_robust_getitem(self) -> None: class UnreliableArrayFailure(Exception): @@ -937,13 +982,8 @@ def multiple_indexing(indexers): ] multiple_indexing(indexers5) - def test_vectorized_indexing_negative_step(self) -> None: - # use dask explicitly when present - open_kwargs: dict[str, Any] | None - if has_dask: - open_kwargs = {"chunks": {}} - else: - open_kwargs = None + def test_vectorized_indexing_negative_step(self, use_dask: bool) -> None: + open_kwargs: dict[str, Any] | None = {"chunks": {}} if use_dask else None in_memory = create_test_data() def multiple_indexing(indexers): @@ -1105,33 +1145,9 @@ def test_roundtrip_empty_vlen_string_array(self) -> None: else: assert np.issubdtype(actual["a"].dtype, np.dtype("=U1")) - @pytest.mark.parametrize( - "decoded_fn, encoded_fn", - [ - ( - create_unsigned_masked_scaled_data, - create_encoded_unsigned_masked_scaled_data, - ), - pytest.param( - create_bad_unsigned_masked_scaled_data, - create_bad_encoded_unsigned_masked_scaled_data, - marks=pytest.mark.xfail(reason="Bad _Unsigned attribute."), - ), - ( - create_signed_masked_scaled_data, - create_encoded_signed_masked_scaled_data, - ), - ( - create_unsigned_false_masked_scaled_data, - create_encoded_unsigned_false_masked_scaled_data, - ), - (create_masked_and_scaled_data, create_encoded_masked_and_scaled_data), - ], - ) + @pytest.mark.parametrize("decoded_fn, encoded_fn", MASK_AND_SCALE_DATA) @pytest.mark.parametrize("dtype", [np.dtype("float64"), np.dtype("float32")]) def test_roundtrip_mask_and_scale(self, decoded_fn, encoded_fn, dtype) -> None: - if hasattr(self, "DIMENSION_KEY") and dtype == np.float32: - pytest.skip("float32 will be treated as float64 in zarr") decoded = decoded_fn(dtype) encoded = encoded_fn(dtype) if decoded["x"].encoding["dtype"] == "u1" and not ( @@ -1392,7 +1408,6 @@ def equals_latlon(obj): assert "coordinates" not in ds["lon"].encoding def test_roundtrip_endian(self) -> None: - skip_if_zarr_format_3("zarr v3 has not implemented endian support yet") ds = Dataset( { "x": np.arange(3, 10, dtype=">i2"), @@ -1456,8 +1471,6 @@ def test_encoding_kwarg(self) -> None: pass def test_encoding_unlimited_dims(self) -> None: - if isinstance(self, ZarrBase): - pytest.skip("No unlimited_dims handled in zarr.") ds = Dataset({"x": ("y", np.arange(10.0))}) with self.roundtrip(ds, save_kwargs=dict(unlimited_dims=["y"])) as actual: assert actual.encoding["unlimited_dims"] == set("y") @@ -1685,6 +1698,25 @@ def test_refresh_from_disk(self) -> None: a.close() b.close() + @pytest.fixture + def byte_attrs_dataset(self) -> dict[str, Any]: + """For testing issue #9407""" + null_byte = b"\x00" + other_bytes = bytes(range(1, 256)) + ds = Dataset({"x": 1}, coords={"x_coord": [1]}) + ds["x"].attrs["null_byte"] = null_byte + ds["x"].attrs["other_bytes"] = other_bytes + + expected = ds.copy() + expected["x"].attrs["null_byte"] = "" + expected["x"].attrs["other_bytes"] = other_bytes.decode(errors="replace") + + return { + "input": ds, + "expected": expected, + "h5netcdf_error": r"Invalid value provided for attribute .*: .*\. Null characters .*", + } + def test_byte_attrs(self, byte_attrs_dataset: dict[str, Any]) -> None: # test for issue #9407 input = byte_attrs_dataset["input"] @@ -2650,6 +2682,32 @@ class ZarrBase(CFEncodedBase): DIMENSION_KEY = "_ARRAY_DIMENSIONS" version_kwargs: dict[str, Any] = {} + @pytest.mark.parametrize("decoded_fn, encoded_fn", MASK_AND_SCALE_DATA) + @pytest.mark.parametrize( + "dtype", + [ + np.dtype("float64"), + pytest.param( + np.dtype("float32"), + marks=pytest.mark.skip( + reason="float32 will be treated as float64 in zarr" + ), + ), + ], + ) + def test_roundtrip_mask_and_scale(self, decoded_fn, encoded_fn, dtype) -> None: + super().test_roundtrip_mask_and_scale(decoded_fn, encoded_fn, dtype) + + @pytest.mark.skip(reason="No unlimited_dims handled in zarr.") + def test_encoding_unlimited_dims(self) -> None: + super().test_encoding_unlimited_dims() + + @skip_if_zarr_format_3( + "endian support requires zarr-python>=3.1", condition=not has_zarr_v3_dtypes + ) + def test_roundtrip_endian(self) -> None: + super().test_roundtrip_endian() + def create_zarr_target(self): raise NotImplementedError @@ -2941,11 +2999,10 @@ def test_shard_encoding(self) -> None: pass @requires_dask + @skip_if_zarr_format_2("sharding requires zarr v3 format") def test_shard_encoding_with_dask(self) -> None: # Test that dask chunks must align with shard boundaries. # See https://github.com/pydata/xarray/issues/10831 - if zarr.config.config["default_zarr_format"] != 3: - pytest.skip("sharding requires zarr v3 format") ds = xr.DataArray(np.arange(12), dims="x", name="var1").to_dataset() @@ -3067,9 +3124,8 @@ def test_drop_encoding(self): with self.create_zarr_target() as store: ds.to_zarr(store, encoding=encodings) + @skip_if_zarr_format_3("This test is unnecessary; no hidden Zarr keys") def test_hidden_zarr_keys(self) -> None: - skip_if_zarr_format_3("This test is unnecessary; no hidden Zarr keys") - expected = create_test_data() with self.create_store() as store: expected.dump_to_store(store) @@ -3096,9 +3152,8 @@ def test_hidden_zarr_keys(self) -> None: with xr.decode_cf(store): pass + @skip_if_zarr_format_2("No dimension names in V2") def test_dimension_names(self) -> None: - skip_if_zarr_format_2("No dimension names in V2") - expected = create_test_data() with self.create_store() as store: expected.dump_to_store(store) @@ -3293,11 +3348,11 @@ def test_append_with_existing_encoding_raises(self) -> None: **self.version_kwargs, ) + @skip_if_zarr_format_3( + "This actually works fine with Zarr format 3", condition=not has_zarr_v3_dtypes + ) @pytest.mark.parametrize("dtype", ["U", "S"]) def test_append_string_length_mismatch_raises(self, dtype) -> None: - if not has_zarr_v3_dtypes: - skip_if_zarr_format_3("This actually works fine with Zarr format 3") - ds, ds_to_append = create_append_string_length_mismatch_test_data(dtype) with self.create_zarr_target() as store_target: ds.to_zarr(store_target, mode="w", **self.version_kwargs) @@ -3306,12 +3361,14 @@ def test_append_string_length_mismatch_raises(self, dtype) -> None: store_target, append_dim="time", **self.version_kwargs ) + @pytest.mark.skipif( + has_zarr_v3_dtypes, + reason="This works on pre ZDtype Zarr-Python, but fails after.", + ) + # ...but it probably would work with Zarr format 2 if we used object dtype + @skip_if_zarr_format_2("This doesn't work with Zarr format 2") @pytest.mark.parametrize("dtype", ["U", "S"]) def test_append_string_length_mismatch_works(self, dtype) -> None: - skip_if_zarr_format_2("This doesn't work with Zarr format 2") - # ...but it probably would if we used object dtype - if has_zarr_v3_dtypes: - pytest.skip("This works on pre ZDtype Zarr-Python, but fails after.") ds, ds_to_append = create_append_string_length_mismatch_test_data(dtype) expected = xr.concat([ds, ds_to_append], dim="time") @@ -3462,12 +3519,9 @@ def test_to_zarr_append_compute_false_roundtrip(self) -> None: with self.open(store) as actual: assert_identical(xr.concat([ds, ds_to_append], dim="time"), actual) - @pytest.mark.parametrize("chunk", [False, True]) - def test_save_emptydim(self, chunk) -> None: - if chunk and not has_dask: - pytest.skip("requires dask") + def test_save_emptydim(self, use_dask) -> None: ds = Dataset({"x": (("a", "b"), np.empty((5, 0))), "y": ("a", [1, 2, 5, 8, 9])}) - if chunk: + if use_dask: ds = ds.chunk({}) # chunk dataset to save dask array with self.roundtrip(ds) as ds_reload: assert_identical(ds, ds_reload) @@ -3486,12 +3540,11 @@ def test_no_warning_from_open_emptydim_with_chunks(self) -> None: assert_identical(ds, ds_reload) @pytest.mark.parametrize("consolidated", [False, True, None]) - @pytest.mark.parametrize("compute", [False, True]) - @pytest.mark.parametrize("use_dask", [False, True]) + @pytest.mark.parametrize( + "compute", [pytest.param(False, marks=requires_dask), True] + ) @pytest.mark.parametrize("write_empty", [False, True, None]) def test_write_region(self, consolidated, compute, use_dask, write_empty) -> None: - if (use_dask or not compute) and not has_dask: - pytest.skip("requires dask") zeros = Dataset({"u": (("x",), np.zeros(10))}) nonzeros = Dataset({"u": (("x",), np.arange(1, 11))}) @@ -3895,13 +3948,12 @@ def test_zarr_fill_value_setting(self, dtype): # ``raise_on_invalid=vn in check_encoding_set`` line in zarr.py # ds.foo.encoding["fill_value"] = fv + @skip_if_zarr_format_2("fill_value is only an encoding key for zarr_format 3") def test_zarr_fill_value_in_encoding_on_read(self) -> None: # GH #10269: the Zarr array fill_value should be preserved in the # variable encoding on read, so that it is not lost on round-trip. # `fill_value` is an independent encoding key only for zarr_format 3; # for zarr_format 2 the fill_value is set via `_FillValue`. - if zarr.config.get("default_zarr_format") != 3: - pytest.skip("fill_value is only an encoding key for zarr_format 3") ds = xr.Dataset({"foo": ("x", [1, 2, 3])}) ds.foo.encoding = {"fill_value": -99} @@ -4160,6 +4212,9 @@ async def test_concurrent_load_multiple_objects( @pytest.mark.asyncio @requires_zarr_v3 + @pytest.mark.skip_if_param( + cls_name="Variable", method="sel", reason="Variable doesn't have a .sel method" + ) @pytest.mark.parametrize("cls_name", ["Variable", "DataArray", "Dataset"]) @pytest.mark.parametrize( "indexer, method, target_zarr_class", @@ -4181,12 +4236,14 @@ async def test_concurrent_load_multiple_objects( {"dim2": [1.0, 3.0]}, "sel", "zarr.core.indexing.AsyncOIndex", + marks=requires_zarr_v3_async_oindex, id="outer-sel", ), pytest.param( {"dim2": [1, 3]}, "isel", "zarr.core.indexing.AsyncOIndex", + marks=requires_zarr_v3_async_oindex, id="outer-isel", ), pytest.param( @@ -4196,6 +4253,7 @@ async def test_concurrent_load_multiple_objects( }, "sel", "zarr.core.indexing.AsyncVIndex", + marks=requires_zarr_v3_async_oindex, id="vectorized-sel", ), pytest.param( @@ -4205,6 +4263,7 @@ async def test_concurrent_load_multiple_objects( }, "isel", "zarr.core.indexing.AsyncVIndex", + marks=requires_zarr_v3_async_oindex, id="vectorized-isel", ), ], @@ -4216,17 +4275,6 @@ async def test_indexing( indexer, target_zarr_class, ) -> None: - if not has_zarr_v3_async_oindex and target_zarr_class in ( - "zarr.core.indexing.AsyncOIndex", - "zarr.core.indexing.AsyncVIndex", - ): - pytest.skip( - "current version of zarr does not support orthogonal or vectorized async indexing" - ) - - if cls_name == "Variable" and method == "sel": - pytest.skip("Variable doesn't have a .sel method") - # Each type of indexing ends up calling a different zarr indexing method # They all use a method named .getitem, but on a different internal zarr class def _resolve_class_from_string(class_path: str) -> type[Any]: @@ -4430,6 +4478,7 @@ def test_write_empty( self, consolidated: bool | None, write_empty: bool | None, + use_dask: bool, ) -> None: def assert_expected_files(expected: list[str], store: str) -> None: """Convenience for comparing with actual files written""" @@ -4472,7 +4521,7 @@ def assert_expected_files(expected: list[str], store: str) -> None: ds = xr.Dataset(data_vars={"test": (("Z", "Y", "X"), data)}) - if has_dask: + if use_dask: ds["test"] = ds["test"].chunk(1) encoding = None else: @@ -5343,7 +5392,14 @@ def test_get_variable_list_empty_driver_kwds(self) -> None: assert "mydataset" in list(actual) -@pytest.fixture(params=["scipy", "netcdf4", "h5netcdf", "zarr"]) +@pytest.fixture( + params=[ + pytest.param("scipy", marks=requires_scipy), + pytest.param("netcdf4", marks=requires_netCDF4), + pytest.param("h5netcdf", marks=requires_h5netcdf), + pytest.param("zarr", marks=requires_zarr), + ] +) def readengine(request): return request.param @@ -5388,14 +5444,6 @@ def tmp_store(request, tmp_path): raise ValueError("not supported") -# using pytest.mark.skipif does not work so this a work around -def skip_if_not_engine(engine): - if engine == "netcdf4": - pytest.importorskip("netCDF4") - else: - pytest.importorskip(engine) - - @requires_dask @pytest.mark.filterwarnings("ignore:use make_scale(name) instead") @pytest.mark.skip( @@ -5404,9 +5452,6 @@ def skip_if_not_engine(engine): def test_open_mfdataset_manyfiles( readengine, nfiles, parallel, chunks, file_cache_maxsize ): - # skip certain combinations - skip_if_not_engine(readengine) - randdata = np.random.randn(nfiles) original = Dataset({"foo": ("x", randdata)}) # test standard open_mfdataset approach with too many files @@ -5873,9 +5918,8 @@ def test_open_mfdataset(self) -> None: open_mfdataset("http://some/remote/uri") @requires_fsspec + @requires_aiobotocore def test_open_mfdataset_no_files(self) -> None: - pytest.importorskip("aiobotocore") - # glob is attempted as of #4823, but finds no files with pytest.raises(OSError, match=r"no files"): open_mfdataset("http://some/remote/uri", engine="zarr") @@ -6802,14 +6846,8 @@ def test_dataarray_to_netcdf_no_name_pathlib(self) -> None: @requires_zarr class TestDataArrayToZarr: - def skip_if_zarr_python_3_and_zip_store(self, store) -> None: - if isinstance(store, zarr.storage.ZipStore): - pytest.skip( - reason="zarr-python 3.x doesn't support reopening ZipStore with a new mode." - ) - + @skip_if_zip_store def test_dataarray_to_zarr_no_name(self, tmp_store) -> None: - self.skip_if_zarr_python_3_and_zip_store(tmp_store) original_da = DataArray(np.arange(12).reshape((3, 4))) original_da.to_zarr(tmp_store) @@ -6817,8 +6855,8 @@ def test_dataarray_to_zarr_no_name(self, tmp_store) -> None: with open_dataarray(tmp_store, engine="zarr") as loaded_da: assert_identical(original_da, loaded_da) + @skip_if_zip_store def test_dataarray_to_zarr_with_name(self, tmp_store) -> None: - self.skip_if_zarr_python_3_and_zip_store(tmp_store) original_da = DataArray(np.arange(12).reshape((3, 4)), name="test") original_da.to_zarr(tmp_store) @@ -6826,8 +6864,8 @@ def test_dataarray_to_zarr_with_name(self, tmp_store) -> None: with open_dataarray(tmp_store, engine="zarr") as loaded_da: assert_identical(original_da, loaded_da) + @skip_if_zip_store def test_dataarray_to_zarr_coord_name_clash(self, tmp_store) -> None: - self.skip_if_zarr_python_3_and_zip_store(tmp_store) original_da = DataArray( np.arange(12).reshape((3, 4)), dims=["x", "y"], name="x" ) @@ -6837,8 +6875,8 @@ def test_dataarray_to_zarr_coord_name_clash(self, tmp_store) -> None: with open_dataarray(tmp_store, engine="zarr") as loaded_da: assert_identical(original_da, loaded_da) + @skip_if_zip_store def test_open_dataarray_options(self, tmp_store) -> None: - self.skip_if_zarr_python_3_and_zip_store(tmp_store) data = DataArray(np.arange(5), coords={"y": ("x", range(1, 6))}, dims=["x"]) data.to_zarr(tmp_store) @@ -6848,10 +6886,10 @@ def test_open_dataarray_options(self, tmp_store) -> None: assert_identical(expected, loaded) @requires_dask + @skip_if_zip_store def test_dataarray_to_zarr_compute_false(self, tmp_store) -> None: from dask.delayed import Delayed - skip_if_zarr_format_3(tmp_store) original_da = DataArray(np.arange(12).reshape((3, 4))) output = original_da.to_zarr(tmp_store, compute=False) @@ -6861,6 +6899,7 @@ def test_dataarray_to_zarr_compute_false(self, tmp_store) -> None: assert_identical(original_da, loaded_da) @requires_dask + @skip_if_zip_store def test_dataarray_to_zarr_align_chunks_true(self, tmp_store) -> None: # TODO: Improve data integrity checks when using Dask. # Detecting automatic alignment issues in Dask can be tricky, @@ -6868,7 +6907,6 @@ def test_dataarray_to_zarr_align_chunks_true(self, tmp_store) -> None: # For now, ensure that the parameter is present, but explore # more robust verification methods to confirm data consistency. - skip_if_zarr_format_3(tmp_store) arr = DataArray( np.arange(4), dims=["a"], coords={"a": np.arange(4)}, name="foo" ).chunk(a=(2, 1, 1)) @@ -6989,11 +7027,11 @@ def test_use_cftime_standard_calendar_default_out_of_range(calendar) -> None: with create_tmp_file() as tmp_file: original.to_netcdf(tmp_file) + # load eagerly: the warning is raised again when lazy data is decoded with pytest.warns(SerializationWarning): - ds = open_dataset(tmp_file) - with ds: - assert_identical(expected_x, ds.x) - assert_identical(expected_time, ds.time) + ds = load_dataset(tmp_file) + assert_identical(expected_x, ds.x) + assert_identical(expected_time, ds.time) @requires_cftime @@ -7187,21 +7225,11 @@ def test_extract_zarr_variable_encoding() -> None: ) -@requires_zarr -@requires_fsspec -@pytest.mark.filterwarnings("ignore:deallocating CachingFileManager") -def test_open_fsspec() -> None: +@pytest.fixture +def fsspec_memory_zarr_stores(): + """Write two zarr stores to fsspec's global in-memory filesystem.""" import fsspec - if not ( - ( - hasattr(zarr.storage, "FSStore") - and hasattr(zarr.storage.FSStore, "getitems") - ) # zarr v2 - or hasattr(zarr.storage, "FsspecStore") # zarr v3 - ): - pytest.skip("zarr too old") - ds = open_dataset(os.path.join(os.path.dirname(__file__), "data", "example_1.nc")) m = fsspec.filesystem("memory") @@ -7214,6 +7242,18 @@ def test_open_fsspec() -> None: mm = m.get_mapper("out2.zarr") ds0.to_zarr(mm) # old interface + yield ds, ds0 + + for path in ("out1.zarr", "out2.zarr"): + m.rm(path, recursive=True) + + +@requires_zarr +@requires_fsspec +@pytest.mark.filterwarnings("ignore:deallocating CachingFileManager") +def test_open_fsspec(fsspec_memory_zarr_stores) -> None: + _, ds0 = fsspec_memory_zarr_stores + # single dataset url = "memory://out2.zarr" ds2 = open_dataset(url, engine="zarr") @@ -7224,17 +7264,23 @@ def test_open_fsspec() -> None: ds2 = open_dataset(url, engine="zarr") xr.testing.assert_equal(ds0, ds2) - # open_mfdataset requires dask - if has_dask: - # multi dataset - url = "memory://out*.zarr" - ds2 = open_mfdataset(url, engine="zarr") - xr.testing.assert_equal(xr.concat([ds, ds0], dim="time"), ds2) - # multi dataset with caching - url = "simplecache::memory://out*.zarr" - ds2 = open_mfdataset(url, engine="zarr") - xr.testing.assert_equal(xr.concat([ds, ds0], dim="time"), ds2) +@requires_zarr +@requires_fsspec +@requires_dask +@pytest.mark.filterwarnings("ignore:deallocating CachingFileManager") +def test_open_mfdataset_fsspec(fsspec_memory_zarr_stores) -> None: + ds, ds0 = fsspec_memory_zarr_stores + + # multi dataset + url = "memory://out*.zarr" + ds2 = open_mfdataset(url, engine="zarr") + xr.testing.assert_equal(xr.concat([ds, ds0], dim="time"), ds2) + + # multi dataset with caching + url = "simplecache::memory://out*.zarr" + ds2 = open_mfdataset(url, engine="zarr") + xr.testing.assert_equal(xr.concat([ds, ds0], dim="time"), ds2) @requires_h5netcdf @@ -7548,19 +7594,18 @@ def test_write_file_from_np_str(str_type: type[str | np.str_], tmpdir: str) -> N @requires_zarr @requires_netCDF4 +@pytest.mark.skipif( + NETCDFC_VERSION is not None and NETCDFC_VERSION < Version("4.8.1"), + reason="requires netcdf-c>=4.8.1", +) +# Bug in netcdf-c==4.8.1 (typo: Nan instead of NaN) +# https://github.com/Unidata/netcdf-c/issues/2265 +@pytest.mark.skipif( + platform.system() == "Windows" and NETCDFC_VERSION == Version("4.8.1"), + reason="netcdf-c==4.8.1 has issues on Windows", +) class TestNCZarr: - @property - def netcdfc_version(self): - return Version(nc4.getlibversion().split()[0].split("-development")[0]) - def _create_nczarr(self, filename): - if self.netcdfc_version < Version("4.8.1"): - pytest.skip("requires netcdf-c>=4.8.1") - if platform.system() == "Windows" and self.netcdfc_version == Version("4.8.1"): - # Bug in netcdf-c==4.8.1 (typo: Nan instead of NaN) - # https://github.com/Unidata/netcdf-c/issues/2265 - pytest.skip("netcdf-c==4.8.1 has issues on Windows") - ds = create_test_data() # Drop dim3: netcdf-c does not support dtype=' None: actual = xr.open_zarr(tmp, consolidated=False) assert_identical(expected, actual) + @pytest.mark.skipif( + NETCDFC_VERSION is not None and NETCDFC_VERSION > Version("4.8.1"), + reason="netcdf-c>4.8.1 adds the _ARRAY_DIMENSIONS attribute", + ) @pytest.mark.parametrize("mode", ["a", "r+"]) @pytest.mark.filterwarnings("ignore:.*non-consolidated metadata.*") def test_raise_writing_to_nczarr(self, mode) -> None: - if self.netcdfc_version > Version("4.8.1"): - pytest.skip("netcdf-c>4.8.1 adds the _ARRAY_DIMENSIONS attribute") - with create_tmp_file(suffix=".zarr") as tmp: ds = self._create_nczarr(tmp) with pytest.raises( diff --git a/xarray/tests/test_cftime_offsets.py b/xarray/tests/test_cftime_offsets.py index 50a8ba3da3d..e774a2f71e6 100644 --- a/xarray/tests/test_cftime_offsets.py +++ b/xarray/tests/test_cftime_offsets.py @@ -2,7 +2,7 @@ import warnings from itertools import product, starmap -from typing import TYPE_CHECKING, Literal +from typing import Literal import numpy as np import pandas as pd @@ -37,7 +37,6 @@ from xarray.tests import ( _CFTIME_CALENDARS, assert_no_warnings, - has_cftime, requires_cftime, requires_pandas_3, ) @@ -1492,7 +1491,6 @@ def test_date_range_like(start, freq, cal_src, cal_tgt, use_cftime, exp0, exp_pd @pytest.mark.parametrize( "freq", ("YE", "YS", "YE-MAY", "MS", "ME", "QS", "h", "min", "s") ) -@pytest.mark.parametrize("use_cftime", (True, False)) def test_date_range_like_no_deprecation(freq, use_cftime): # ensure no internal warnings # TODO: remove once freq string deprecation is finished @@ -1547,23 +1545,12 @@ def as_timedelta_not_implemented_error(): tick.as_timedelta() -@pytest.mark.parametrize("use_cftime", [True, False]) def test_cftime_or_date_range_invalid_inclusive_value(use_cftime: bool) -> None: - if use_cftime and not has_cftime: - pytest.skip("requires cftime") - - if TYPE_CHECKING: - pytest.skip("inclusive type checked internally") - with pytest.raises(ValueError, match="nclusive"): - date_range("2000", periods=3, inclusive="foo", use_cftime=use_cftime) + date_range("2000", periods=3, inclusive="foo", use_cftime=use_cftime) # type: ignore[arg-type] -@pytest.mark.parametrize("use_cftime", [True, False]) def test_cftime_or_date_range_inclusive_None(use_cftime: bool) -> None: - if use_cftime and not has_cftime: - pytest.skip("requires cftime") - result_None = date_range("2000-01-01", "2000-01-04", use_cftime=use_cftime) result_both = date_range( "2000-01-01", "2000-01-04", inclusive="both", use_cftime=use_cftime diff --git a/xarray/tests/test_cftimeindex_resample.py b/xarray/tests/test_cftimeindex_resample.py index d40b4a95f2a..87d463e4d1f 100644 --- a/xarray/tests/test_cftimeindex_resample.py +++ b/xarray/tests/test_cftimeindex_resample.py @@ -16,7 +16,7 @@ from xarray.coding.cftimeindex import CFTimeIndex from xarray.core.resample_cftime import CFTimeGrouper from xarray.core.types import PDDatetimeUnitOptions -from xarray.tests import has_pandas_3_1 +from xarray.tests import requires_pandas_3_1 cftime = pytest.importorskip("cftime") @@ -71,7 +71,13 @@ def has_non_tick_resample_freq(freqs): return not has_tick_resample_freq(freqs) -FREQS_WITH_TICK_RESAMPLE_FREQ = list(filter(has_tick_resample_freq, FREQS)) +FREQS_WITH_TICK_RESAMPLE_FREQ = [ + # Resampling to Day is only valid for pandas >= 3.1 + pytest.param(freqs, marks=requires_pandas_3_1) + if isinstance(to_offset(freqs[1]), Day) + else freqs + for freqs in filter(has_tick_resample_freq, FREQS) +] FREQS_WITH_NON_TICK_RESAMPLE_FREQ = list(filter(has_non_tick_resample_freq, FREQS)) @@ -139,9 +145,6 @@ def da(index) -> xr.DataArray: @pytest.mark.parametrize("offset", [None, "5s"], ids=lambda x: f"{x}") def test_resample_with_tick_resample_freq(freqs, closed, label, offset) -> None: initial_freq, resample_freq = freqs - resample_freq_as_offset = to_offset(resample_freq) - if isinstance(resample_freq_as_offset, Day) and not has_pandas_3_1: - pytest.skip("Only valid for pandas >= 3.1") start = "2000-01-01T12:07:01" origin = "start" diff --git a/xarray/tests/test_coarsen.py b/xarray/tests/test_coarsen.py index fb737d73067..fc5e167dcad 100644 --- a/xarray/tests/test_coarsen.py +++ b/xarray/tests/test_coarsen.py @@ -11,7 +11,6 @@ assert_allclose, assert_equal, assert_identical, - has_dask, raise_if_dask_computes, requires_cftime, ) @@ -25,10 +24,9 @@ def test_coarsen_absent_dims_error(ds: Dataset) -> None: ds.coarsen(foo=2) -@pytest.mark.parametrize("dask", [True, False]) @pytest.mark.parametrize(("boundary", "side"), [("trim", "left"), ("pad", "right")]) -def test_coarsen_dataset(ds, dask, boundary, side): - if dask and has_dask: +def test_coarsen_dataset(ds, use_dask, boundary, side): + if use_dask: ds = ds.chunk({"x": 4}) actual = ds.coarsen(time=2, x=3, boundary=boundary, side=side).max() @@ -41,9 +39,8 @@ def test_coarsen_dataset(ds, dask, boundary, side): ) -@pytest.mark.parametrize("dask", [True, False]) -def test_coarsen_coords(ds, dask): - if dask and has_dask: +def test_coarsen_coords(ds, use_dask): + if use_dask: ds = ds.chunk({"x": 4}) # check if coord_func works @@ -241,13 +238,19 @@ def test_coarsen_da_keep_attrs(funcname, argument) -> None: assert result.name == "name" -@pytest.mark.parametrize("da", (1, 2), indirect=True) -@pytest.mark.parametrize("window", (1, 2, 3, 4)) +# da=2 contains NaNs, so window=1 would lead to all-NaN slices +@pytest.mark.parametrize( + "da, window", + [ + (da, window) + for da in (1, 2) + for window in (1, 2, 3, 4) + if (da, window) != (2, 1) + ], + indirect=["da"], +) @pytest.mark.parametrize("name", ("sum", "mean", "std", "max")) def test_coarsen_da_reduce(da, window, name) -> None: - if da.isnull().sum() > 1 and window == 1: - pytest.skip("These parameters lead to all-NaN slices") - # Use boundary="trim" to accommodate all window sizes used in tests coarsen_obj = da.coarsen(time=window, boundary="trim") @@ -258,8 +261,7 @@ def test_coarsen_da_reduce(da, window, name) -> None: class TestCoarsenConstruct: - @pytest.mark.parametrize("dask", [True, False]) - def test_coarsen_construct(self, dask: bool) -> None: + def test_coarsen_construct(self, use_dask: bool) -> None: ds = Dataset( { "vart": ("time", np.arange(48), {"a": "b"}), @@ -271,7 +273,7 @@ def test_coarsen_construct(self, dask: bool) -> None: attrs={"foo": "bar"}, ) - if dask and has_dask: + if use_dask: ds = ds.chunk({"x": 4, "time": 10}) expected = xr.Dataset(attrs={"foo": "bar"}) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 298154ea131..bd31dc108cc 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -1084,16 +1084,19 @@ def test_decode_ambiguous_time_warns(calendar) -> None: @pytest.mark.filterwarnings("ignore:Times can't be serialized faithfully") -@pytest.mark.parametrize("encoding_units", FREQUENCIES_TO_ENCODING_UNITS.values()) @pytest.mark.parametrize("freq", FREQUENCIES_TO_ENCODING_UNITS.keys()) -@pytest.mark.parametrize("use_cftime", [True, False]) +@pytest.mark.parametrize("encoding_units", FREQUENCIES_TO_ENCODING_UNITS.values()) +@pytest.mark.skip_if_param( + use_cftime=True, freq="ns", reason="Nanosecond frequency is not valid for cftime" +) +@pytest.mark.skip_if_param( + use_cftime=True, + encoding_units="nanoseconds", + reason="Nanosecond frequency is not valid for cftime", +) def test_encode_cf_datetime_defaults_to_correct_dtype( encoding_units, freq, use_cftime ) -> None: - if not has_cftime and use_cftime: - pytest.skip("Test requires cftime") - if (freq == "ns" or encoding_units == "nanoseconds") and use_cftime: - pytest.skip("Nanosecond frequency is not valid for cftime dates.") times = date_range("2000", periods=3, freq=freq, use_cftime=use_cftime) units = f"{encoding_units} since 2000-01-01" encoded, _units, _ = encode_cf_datetime(times, units) @@ -1171,12 +1174,13 @@ def test_round_trip_standard_calendar_cftime_datetimes_pre_reform() -> None: @pytest.mark.parametrize("calendar", ["standard", "gregorian"]) +# overrides the time_unit fixture: datetime64[ns] values can only be defined +# post reform +@pytest.mark.parametrize("time_unit", ["s", "ms", "us"]) def test_encode_cf_datetime_gregorian_proleptic_gregorian_mismatch_error( calendar: str, time_unit: PDDatetimeUnitOptions, ) -> None: - if time_unit == "ns": - pytest.skip("datetime64[ns] values can only be defined post reform") dates = np.array(["0001-01-01", "2001-01-01"], dtype=f"datetime64[{time_unit}]") with pytest.raises(ValueError, match="proleptic_gregorian"): encode_cf_datetime(dates, "seconds since 2000-01-01", calendar) @@ -1268,12 +1272,8 @@ def test_decode_cf_datetime_uint64_with_cftime_overflow_error(): decode_cf_datetime(num_dates, units, calendar) -@pytest.mark.parametrize("use_cftime", [True, False]) def test_decode_0size_datetime(use_cftime): # GH1329 - if use_cftime and not has_cftime: - pytest.skip() - dtype = object if use_cftime else "=M8[ns]" expected = np.array([], dtype=dtype) actual = decode_cf_datetime( @@ -1712,50 +1712,56 @@ def test_encode_cf_datetime_cftime_datetime_via_dask(units, dtype) -> None: np.testing.assert_equal(decoded_times, times) -@pytest.mark.parametrize( - "use_cftime", [False, pytest.param(True, marks=requires_cftime)] -) -@pytest.mark.parametrize("use_dask", [False, pytest.param(True, marks=requires_dask)]) -def test_encode_cf_datetime_units_change(use_cftime, use_dask) -> None: +def test_encode_cf_datetime_units_change(use_cftime) -> None: times = date_range(start="2000", freq="12h", periods=3, use_cftime=use_cftime) encoding = dict(units="days since 2000-01-01", dtype=np.dtype("int64")) variable = Variable(["time"], times, encoding=encoding) - if use_dask: - variable = variable.chunk({"time": 1}) - with pytest.raises(ValueError, match="Times can't be serialized"): - conventions.encode_cf_variable(variable).compute() + with pytest.warns(UserWarning, match="Times can't be serialized"): + encoded = conventions.encode_cf_variable(variable) + if use_cftime: + expected_units = "hours since 2000-01-01 00:00:00.000000" else: - with pytest.warns(UserWarning, match="Times can't be serialized"): - encoded = conventions.encode_cf_variable(variable) - if use_cftime: - expected_units = "hours since 2000-01-01 00:00:00.000000" - else: - expected_units = "hours since 2000-01-01" - assert encoded.attrs["units"] == expected_units - decoded = conventions.decode_cf_variable( - "name", encoded, decode_times=CFDatetimeCoder(use_cftime=use_cftime) - ) - assert_equal(variable, decoded) + expected_units = "hours since 2000-01-01" + assert encoded.attrs["units"] == expected_units + decoded = conventions.decode_cf_variable( + "name", encoded, decode_times=CFDatetimeCoder(use_cftime=use_cftime) + ) + assert_equal(variable, decoded) -@pytest.mark.parametrize("use_dask", [False, pytest.param(True, marks=requires_dask)]) -def test_encode_cf_datetime_precision_loss_regression_test(use_dask) -> None: +@requires_dask +def test_encode_cf_datetime_units_change_dask(use_cftime) -> None: + # With dask the units cannot be changed on the fly, so encoding raises + times = date_range(start="2000", freq="12h", periods=3, use_cftime=use_cftime) + encoding = dict(units="days since 2000-01-01", dtype=np.dtype("int64")) + variable = Variable(["time"], times, encoding=encoding).chunk({"time": 1}) + + with pytest.raises(ValueError, match="Times can't be serialized"): + conventions.encode_cf_variable(variable).compute() + + +def test_encode_cf_datetime_precision_loss_regression_test() -> None: # Regression test for # https://github.com/pydata/xarray/issues/9134#issuecomment-2191446463 times = date_range("2000", periods=5, freq="ns") encoding = dict(units="seconds since 1970-01-01", dtype=np.dtype("int64")) variable = Variable(["time"], times, encoding=encoding) - if use_dask: - variable = variable.chunk({"time": 1}) - with pytest.raises(ValueError, match="Times can't be serialized"): - conventions.encode_cf_variable(variable).compute() - else: - with pytest.warns(UserWarning, match="Times can't be serialized"): - encoded = conventions.encode_cf_variable(variable) - decoded = conventions.decode_cf_variable("name", encoded) - assert_equal(variable, decoded) + with pytest.warns(UserWarning, match="Times can't be serialized"): + encoded = conventions.encode_cf_variable(variable) + decoded = conventions.decode_cf_variable("name", encoded) + assert_equal(variable, decoded) + + +@requires_dask +def test_encode_cf_datetime_precision_loss_regression_test_dask() -> None: + times = date_range("2000", periods=5, freq="ns") + encoding = dict(units="seconds since 1970-01-01", dtype=np.dtype("int64")) + variable = Variable(["time"], times, encoding=encoding).chunk({"time": 1}) + + with pytest.raises(ValueError, match="Times can't be serialized"): + conventions.encode_cf_variable(variable).compute() @requires_dask @@ -1787,29 +1793,33 @@ def test_encode_cf_timedelta_via_dask( assert decoded_times.dtype == times.dtype -@pytest.mark.parametrize("use_dask", [False, pytest.param(True, marks=requires_dask)]) -def test_encode_cf_timedelta_units_change(use_dask) -> None: +def test_encode_cf_timedelta_units_change() -> None: timedeltas = pd.timedelta_range(start="0h", freq="12h", periods=3) encoding = dict(units="days", dtype=np.dtype("int64")) variable = Variable(["time"], timedeltas, encoding=encoding) - if use_dask: - variable = variable.chunk({"time": 1}) - with pytest.raises(ValueError, match="Timedeltas can't be serialized"): - conventions.encode_cf_variable(variable).compute() - else: - # In this case we automatically modify the encoding units to continue - # encoding with integer values. - with pytest.warns(UserWarning, match="Timedeltas can't be serialized"): - encoded = conventions.encode_cf_variable(variable) - assert encoded.attrs["units"] == "hours" - decoded = conventions.decode_cf_variable( - "name", encoded, decode_timedelta=CFTimedeltaCoder(time_unit="ns") - ) - assert_equal(variable, decoded) + # In this case we automatically modify the encoding units to continue + # encoding with integer values. + with pytest.warns(UserWarning, match="Timedeltas can't be serialized"): + encoded = conventions.encode_cf_variable(variable) + assert encoded.attrs["units"] == "hours" + decoded = conventions.decode_cf_variable( + "name", encoded, decode_timedelta=CFTimedeltaCoder(time_unit="ns") + ) + assert_equal(variable, decoded) + + +@requires_dask +def test_encode_cf_timedelta_units_change_dask() -> None: + # With dask the units cannot be changed on the fly, so encoding raises + timedeltas = pd.timedelta_range(start="0h", freq="12h", periods=3) + encoding = dict(units="days", dtype=np.dtype("int64")) + variable = Variable(["time"], timedeltas, encoding=encoding).chunk({"time": 1}) + + with pytest.raises(ValueError, match="Timedeltas can't be serialized"): + conventions.encode_cf_variable(variable).compute() -@pytest.mark.parametrize("use_dask", [False, pytest.param(True, marks=requires_dask)]) def test_encode_cf_timedelta_small_dtype_missing_value(use_dask) -> None: # Regression test for GitHub issue #9134 timedeltas = np.array([1, 2, "NaT", 4], dtype="timedelta64[D]").astype( @@ -1975,9 +1985,7 @@ def test_lazy_decode_timedelta_error() -> None: "calendar", [ "standard", - pytest.param( - "360_day", marks=pytest.mark.skipif(not has_cftime, reason="no cftime") - ), + pytest.param("360_day", marks=requires_cftime), ], ) def test_duck_array_decode_times(calendar) -> None: diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index ecdc7eb6e49..0c5868229bd 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -26,7 +26,6 @@ from xarray.tests import ( dask_array_api, dask_array_type, - has_dask, raise_if_dask_computes, requires_cftime, requires_dask, @@ -2109,11 +2108,7 @@ def apply_truncate_x_x_valid(obj): ) -@pytest.mark.parametrize("use_dask", [True, False]) def test_dot(use_dask: bool) -> None: - if use_dask and not has_dask: - pytest.skip("test for dask.") - a = np.arange(30 * 4).reshape(30, 4) b = np.arange(30 * 4 * 5).reshape(30, 4, 5) c = np.arange(5 * 60).reshape(5, 60) @@ -2238,13 +2233,9 @@ def test_dot(use_dask: bool) -> None: pickle.loads(pickle.dumps(xr.dot(da_a))) -@pytest.mark.parametrize("use_dask", [True, False]) def test_dot_align_coords(use_dask: bool) -> None: # GH 3694 - if use_dask and not has_dask: - pytest.skip("test for dask.") - a = np.arange(30 * 4).reshape(30, 4) b = np.arange(30 * 4 * 5).reshape(30, 4, 5) @@ -2385,9 +2376,6 @@ def test_where_attrs() -> None: assert_identical(ds_expected, ds_actual) -@pytest.mark.parametrize( - "use_dask", [pytest.param(False, id="nodask"), pytest.param(True, id="dask")] -) @pytest.mark.parametrize( ["x", "coeffs", "expected"], [ @@ -2522,8 +2510,6 @@ def test_polyval( expected: xr.DataArray | xr.Dataset, ) -> None: if use_dask: - if not has_dask: - pytest.skip("requires dask") coeffs = coeffs.chunk({"degree": 2}) x = x.chunk({"x": 2}) @@ -2534,9 +2520,6 @@ def test_polyval( @requires_cftime -@pytest.mark.parametrize( - "use_dask", [pytest.param(False, id="nodask"), pytest.param(True, id="dask")] -) @pytest.mark.parametrize("date", ["1970-01-01", "0753-04-21"]) def test_polyval_cftime(use_dask: bool, date: str) -> None: import cftime @@ -2548,8 +2531,6 @@ def test_polyval_cftime(use_dask: bool, date: str) -> None: coeffs = xr.DataArray([0, 1], dims="degree", coords={"degree": [0, 1]}) if use_dask: - if not has_dask: - pytest.skip("requires dask") coeffs = coeffs.chunk({"degree": 2}) x = x.chunk({"x": 2}) @@ -2587,9 +2568,6 @@ def test_polyval_timedelta_nat() -> None: assert not np.isnan(actual.values[[0, 2]]).any() -@pytest.mark.parametrize( - "use_dask", [pytest.param(False, id="nodask"), pytest.param(True, id="dask")] -) @pytest.mark.parametrize( "x", [ @@ -2626,8 +2604,6 @@ def test_polyfit_polyval_integration( ) -> None: y.coords["x"] = x if use_dask: - if not has_dask: - pytest.skip("requires dask") y = y.chunk({"x": 2}) fit = y.polyfit(dim="x", deg=2) @@ -2636,7 +2612,6 @@ def test_polyfit_polyval_integration( xr.testing.assert_allclose(evaluated.variable, expected.variable) -@pytest.mark.parametrize("use_dask", [False, True]) @pytest.mark.parametrize( "a, b, ae, be, dim, axis", [ @@ -2738,8 +2713,6 @@ def test_cross(a, b, ae, be, dim: str, axis: int, use_dask: bool) -> None: expected = np.cross(ae, be, axis=axis) if use_dask: - if not has_dask: - pytest.skip("test for dask.") a = a.chunk() b = b.chunk() diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 7db24b91dca..f0dc2de2a3e 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -50,7 +50,6 @@ assert_no_warnings, dask_array_api, dask_array_type, - has_dask, has_dask_ge_2025_1_0, has_pyarrow, raise_if_dask_computes, @@ -561,6 +560,18 @@ def should_add_coord_to_array(self, name, var, dims): assert_identical(actual.coords, coords, check_default_indexes=False) assert "x_bnds" not in actual.dims + def test_replace_maybe_drop_dims_keeps_reordered_coords(self) -> None: + array = DataArray( + np.empty((0, 2)), + dims=("x", "y"), + coords={"x": [], "y": [1, 1]}, + ) + + actual = array._replace_maybe_drop_dims(Variable(("y", "x"), np.empty((1, 0)))) + + assert "x" in actual.coords + assert actual.sizes == {"y": 1, "x": 0} + def test_replace_maybe_drop_dims_preserves_multi_coord_index(self) -> None: # Regression test for https://github.com/pydata/xarray/issues/11215 # Multi-coordinate indexes spanning multiple dims should be preserved @@ -3981,21 +3992,15 @@ def test_to_and_from_empty_series(self) -> None: def test_series_categorical_index(self) -> None: # regression test for GH700 - if not hasattr(pd, "CategoricalIndex"): - pytest.skip("requires pandas with CategoricalIndex") - s = pd.Series(np.arange(5), index=pd.CategoricalIndex(list("aabbc"))) arr = DataArray(s) assert "a a b b" in repr(arr) # should not error - @pytest.mark.parametrize("use_dask", [True, False]) @pytest.mark.parametrize("data", ["list", "array", True]) @pytest.mark.parametrize("encoding", [True, False]) def test_to_and_from_dict( self, encoding: bool, data: bool | Literal["list", "array"], use_dask: bool ) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") encoding_data = {"bar": "spam"} array = DataArray( np.random.randn(2, 3), {"x": ["a", "b"]}, ["x", "y"], name="foo" @@ -4018,10 +4023,7 @@ def test_to_and_from_dict( if encoding: expected["encoding"] = encoding_data - if has_dask: - da = array.chunk() - else: - da = array + da = array.chunk() if use_dask else array if data == "array" or data is False: with raise_if_dask_computes(): @@ -4335,6 +4337,13 @@ def test_roll_no_coords(self) -> None: expected = DataArray([3, 1, 2], coords=[("x", [0, 1, 2])]) assert_identical(expected, actual) + @pytest.mark.parametrize("shift", [-1, 0, 1]) + @pytest.mark.parametrize("roll_coords", [False, True]) + def test_roll_empty(self, shift: int, roll_coords: bool) -> None: + arr = DataArray([], coords={"x": range(0)}, dims="x") + actual = arr.roll(x=shift, roll_coords=roll_coords) + assert_identical(arr, actual) + def test_copy_with_data(self) -> None: orig = DataArray( np.random.random(size=(2, 2)), @@ -4664,12 +4673,9 @@ def test_rank(self) -> None: y = DataArray([0.75, 0.25, np.nan, 0.5, 1.0], dims=("z",)) assert_equal(y.rank("z", pct=True), y) - @pytest.mark.parametrize("use_dask", [True, False]) @pytest.mark.parametrize("use_datetime", [True, False]) @pytest.mark.filterwarnings("ignore:overflow encountered in multiply") def test_polyfit(self, use_dask, use_datetime) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") xcoord = xr.DataArray( pd.date_range("1970-01-01", freq="D", periods=10), dims=("x",), name="x" ) @@ -4888,11 +4894,8 @@ def test_pad_keep_attrs(self, keep_attrs, attrs, expected) -> None: @pytest.mark.parametrize( "engine", ["python", None, pytest.param("numexpr", marks=[requires_numexpr])] ) - @pytest.mark.parametrize( - "backend", ["numpy", pytest.param("dask", marks=[requires_dask])] - ) def test_query( - self, backend, engine: QueryEngineOptions, parser: QueryParserOptions + self, use_dask: bool, engine: QueryEngineOptions, parser: QueryParserOptions ) -> None: """Test querying a dataset.""" @@ -4909,7 +4912,7 @@ def test_query( cc = DataArray(data=c, dims=["y"], name="c", coords={"c2": ("y", c)}) dd = DataArray(data=d, dims=["z"], name="d", coords={"d2": ("z", d)}) - if backend == "dask": + if use_dask: da = dask_array_api aa = aa.copy(data=da.from_array(a, chunks=3)) bb = bb.copy(data=da.from_array(b, chunks=3)) @@ -4958,11 +4961,7 @@ def test_query( aa.query(x="spam > 50") # name not present @requires_scipy - @pytest.mark.parametrize("use_dask", [True, False]) def test_curvefit(self, use_dask) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") - def exp_decay(t, n0, tau=1): return n0 * np.exp(-t / tau) @@ -5029,11 +5028,7 @@ def exp_decay(t, n0, tau=1): assert params == param_names @requires_scipy - @pytest.mark.parametrize("use_dask", [True, False]) def test_curvefit_multidimensional_guess(self, use_dask: bool) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") - def sine(t, a, f, p): return a * np.sin(2 * np.pi * (f * t + p)) @@ -5079,11 +5074,7 @@ def sine(t, a, f, p): ) @requires_scipy - @pytest.mark.parametrize("use_dask", [True, False]) def test_curvefit_multidimensional_bounds(self, use_dask: bool) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") - def sine(t, a, f, p): return a * np.sin(2 * np.pi * (f * t + p)) @@ -5141,11 +5132,7 @@ def sine(t, a, f, p): ) @requires_scipy - @pytest.mark.parametrize("use_dask", [True, False]) def test_curvefit_ignore_errors(self, use_dask: bool) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") - # nonsense function to make the optimization fail def line(x, a, b): if a > 10: @@ -5375,15 +5362,6 @@ def test_argmax( assert_identical(result2, expected2) - @pytest.mark.parametrize( - "use_dask", - [ - pytest.param( - True, marks=pytest.mark.skipif(not has_dask, reason="no dask") - ), - False, - ], - ) def test_idxmin( self, x: np.ndarray, @@ -5497,7 +5475,6 @@ def test_idxmin( result7 = ar0.idxmin(fill_value=-1j) assert_identical(result7, expected7) - @pytest.mark.parametrize("use_dask", [True, False]) def test_idxmax( self, x: np.ndarray, @@ -5506,8 +5483,6 @@ def test_idxmax( nanindex: int | None, use_dask: bool, ) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") if use_dask and x.dtype.kind == "M": pytest.xfail("dask operation 'argmax' breaks when dtype is datetime64 (M)") ar0_raw = xr.DataArray( @@ -5979,9 +5954,6 @@ def test_argmax( assert_identical(result3, expected2) - @pytest.mark.parametrize( - "use_dask", [pytest.param(True, id="dask"), pytest.param(False, id="nodask")] - ) def test_idxmin( self, x: np.ndarray, @@ -5990,8 +5962,6 @@ def test_idxmin( nanindex: list[int | None], use_dask: bool, ) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") if use_dask and x.dtype.kind == "M": pytest.xfail("dask operation 'argmin' breaks when dtype is datetime64 (M)") @@ -6124,9 +6094,6 @@ def test_idxmin( result7 = ar0.idxmin(dim="x", fill_value=-5j) assert_identical(result7, expected7) - @pytest.mark.parametrize( - "use_dask", [pytest.param(True, id="dask"), pytest.param(False, id="nodask")] - ) def test_idxmax( self, x: np.ndarray, @@ -6135,8 +6102,6 @@ def test_idxmax( nanindex: list[int | None], use_dask: bool, ) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") if use_dask and x.dtype.kind == "M": pytest.xfail("dask operation 'argmax' breaks when dtype is datetime64 (M)") @@ -7067,10 +7032,8 @@ def test_argmax_dim( class TestReduceND(TestReduce): @pytest.mark.parametrize("op", ["idxmin", "idxmax"]) @pytest.mark.parametrize("ndim", [3, 5]) + @requires_dask def test_idxminmax_dask(self, op: str, ndim: int) -> None: - if not has_dask: - pytest.skip("requires dask") - ar0_raw = xr.DataArray( np.random.random_sample(size=[10] * ndim), dims=list("abcdefghij"[: ndim - 1]) + ["x"], diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 3c4c9edcb34..1f622d9b05a 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -62,8 +62,6 @@ create_test_data, dask_array_api, dask_array_type, - has_cftime, - has_dask, has_pyarrow, raise_if_dask_computes, requires_bottleneck, @@ -1152,9 +1150,9 @@ def test_chunks_does_not_load_data(self) -> None: "use_cftime,calendar", [ (False, "standard"), - (pytest.param(True, marks=pytest.mark.skipif(not has_cftime)), "standard"), - (pytest.param(True, marks=pytest.mark.skipif(not has_cftime)), "noleap"), - (pytest.param(True, marks=pytest.mark.skipif(not has_cftime)), "360_day"), + pytest.param(True, "standard", marks=requires_cftime), + pytest.param(True, "noleap", marks=requires_cftime), + pytest.param(True, "360_day", marks=requires_cftime), ], ) def test_chunk_by_season_resampler(self, use_cftime: bool, calendar: str) -> None: @@ -1329,7 +1327,7 @@ def get_dask_names(ds): "standard", pytest.param( "gregorian", - marks=pytest.mark.skipif(not has_cftime, reason="needs cftime"), + marks=requires_cftime, ), ), ) @@ -7748,10 +7746,7 @@ def test_astype_attrs(self) -> None: @pytest.mark.parametrize( "engine", ["python", None, pytest.param("numexpr", marks=[requires_numexpr])] ) - @pytest.mark.parametrize( - "backend", ["numpy", pytest.param("dask", marks=[requires_dask])] - ) - def test_query(self, backend, engine, parser) -> None: + def test_query(self, use_dask: bool, engine, parser) -> None: """Test querying a dataset.""" # setup test data @@ -7764,7 +7759,7 @@ def test_query(self, backend, engine, parser) -> None: ) e = np.arange(0, 10 * 20).reshape(10, 20) f = np.random.normal(0, 1, size=(10, 20, 30)) - if backend == "numpy": + if not use_dask: ds = Dataset( { "a": ("x", a), @@ -7783,7 +7778,7 @@ def test_query(self, backend, engine, parser) -> None: "f2": (("x", "y", "z"), f), }, ) - elif backend == "dask": + else: da = dask_array_api ds = Dataset( { @@ -7918,7 +7913,7 @@ def test_query(self, backend, engine, parser) -> None: @pytest.mark.parametrize("test_elements", ([1, 2], np.array([1, 2]), DataArray([1, 2]))) -def test_isin(test_elements, backend) -> None: +def test_isin(test_elements, use_dask: bool) -> None: expected = Dataset( data_vars={ "var1": (("dim1",), [0, 1]), @@ -7927,7 +7922,7 @@ def test_isin(test_elements, backend) -> None: } ).astype("bool") - if backend == "dask": + if use_dask: expected = expected.chunk() result = Dataset( @@ -8024,9 +8019,8 @@ def test_raise_no_warning_assert_close(ds) -> None: assert_allclose(ds, ds) -@pytest.mark.parametrize("dask", [True, False]) @pytest.mark.parametrize("edge_order", [1, 2]) -def test_differentiate(dask, edge_order) -> None: +def test_differentiate(use_dask, edge_order) -> None: rs = np.random.default_rng(42) coord = [0.2, 0.35, 0.4, 0.6, 0.7, 0.75, 0.76, 0.8] @@ -8035,7 +8029,7 @@ def test_differentiate(dask, edge_order) -> None: dims=["x", "y"], coords={"x": coord, "z": 3, "x2d": (("x", "y"), rs.random((8, 6)))}, ) - if dask and has_dask: + if use_dask: da = da.chunk({"x": 4}) ds = xr.Dataset({"var": da}) @@ -8073,8 +8067,7 @@ def test_differentiate(dask, edge_order) -> None: da.differentiate("x2d") -@pytest.mark.parametrize("dask", [True, False]) -def test_differentiate_datetime(dask) -> None: +def test_differentiate_datetime(use_dask) -> None: rs = np.random.default_rng(42) coord = np.array( [ @@ -8095,7 +8088,7 @@ def test_differentiate_datetime(dask) -> None: dims=["x", "y"], coords={"x": coord, "z": 3, "x2d": (("x", "y"), rs.random((8, 6)))}, ) - if dask and has_dask: + if use_dask: da = da.chunk({"x": 4}) # along x @@ -8123,8 +8116,7 @@ def test_differentiate_datetime(dask) -> None: @requires_cftime -@pytest.mark.parametrize("dask", [True, False]) -def test_differentiate_cftime(dask) -> None: +def test_differentiate_cftime(use_dask) -> None: rs = np.random.default_rng(42) coord = xr.date_range("2000", periods=8, freq="2ME", use_cftime=True) @@ -8134,7 +8126,7 @@ def test_differentiate_cftime(dask) -> None: dims=["time", "y"], ) - if dask and has_dask: + if use_dask: da = da.chunk({"time": 4}) actual = da.differentiate("time", edge_order=1, datetime_unit="D") @@ -8152,8 +8144,7 @@ def test_differentiate_cftime(dask) -> None: assert_allclose(actual, xr.ones_like(da["time"]).astype(float)) -@pytest.mark.parametrize("dask", [True, False]) -def test_integrate(dask) -> None: +def test_integrate(use_dask) -> None: rs = np.random.default_rng(42) coord = [0.2, 0.35, 0.4, 0.6, 0.7, 0.75, 0.76, 0.8] @@ -8167,7 +8158,7 @@ def test_integrate(dask) -> None: "x2d": (("x", "y"), rs.random((8, 6))), }, ) - if dask and has_dask: + if use_dask: da = da.chunk({"x": 4}) ds = xr.Dataset({"var": da}) @@ -8206,8 +8197,7 @@ def test_integrate(dask) -> None: @requires_scipy -@pytest.mark.parametrize("dask", [True, False]) -def test_cumulative_integrate(dask) -> None: +def test_cumulative_integrate(use_dask) -> None: rs = np.random.default_rng(43) coord = [0.2, 0.35, 0.4, 0.6, 0.7, 0.75, 0.76, 0.8] @@ -8221,7 +8211,7 @@ def test_cumulative_integrate(dask) -> None: "x2d": (("x", "y"), rs.random((8, 6))), }, ) - if dask and has_dask: + if use_dask: da = da.chunk({"x": 4}) ds = xr.Dataset({"var": da}) @@ -8267,12 +8257,10 @@ def test_cumulative_integrate(dask) -> None: da.cumulative_integrate("x2d") -@pytest.mark.parametrize("dask", [True, False]) -@pytest.mark.parametrize("which_datetime", ["np", "cftime"]) -def test_trapezoid_datetime(dask, which_datetime) -> None: +def test_trapezoid_datetime(use_dask, use_cftime) -> None: rs = np.random.default_rng(42) coord: ArrayLike - if which_datetime == "np": + if not use_cftime: coord = np.array( [ "2004-07-13", @@ -8287,8 +8275,6 @@ def test_trapezoid_datetime(dask, which_datetime) -> None: dtype="datetime64", ) else: - if not has_cftime: - pytest.skip("Test requires cftime.") coord = xr.date_range("2000", periods=8, freq="2D", use_cftime=True) da = xr.DataArray( @@ -8297,7 +8283,7 @@ def test_trapezoid_datetime(dask, which_datetime) -> None: dims=["time", "y"], ) - if dask and has_dask: + if use_dask: da = da.chunk({"time": 4}) actual = da.integrate("time", datetime_unit="D") diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 55ac0015db5..cbced0cd0d5 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -36,11 +36,10 @@ assert_identical, dask_array_api, dask_array_type, - has_h5netcdf, - has_netCDF4, - has_scipy, requires_cftime, + requires_h5netcdf, requires_netCDF4, + requires_scipy, requires_zarr, ) from xarray.tests.test_backends import ( @@ -55,13 +54,11 @@ def tmp_netcdf_filename(tmpdir): return str(tmpdir.join("testfile.nc")) -ENGINES = [] -if has_scipy: - ENGINES.append("scipy") -if has_netCDF4: - ENGINES.append("netcdf4") -if has_h5netcdf: - ENGINES.append("h5netcdf") +ENGINE_MARKS = { + "scipy": requires_scipy, + "netcdf4": requires_netCDF4, + "h5netcdf": requires_h5netcdf, +} NC_FORMATS = { "netcdf4": [ @@ -76,11 +73,14 @@ def tmp_netcdf_filename(tmpdir): } ENGINES_AND_FORMATS = [ - ("netcdf4", "NETCDF3_CLASSIC"), - ("netcdf4", "NETCDF4_CLASSIC"), - ("netcdf4", "NETCDF4"), - ("h5netcdf", "NETCDF4"), - ("scipy", "NETCDF3_64BIT"), + pytest.param(engine, nc_format, marks=ENGINE_MARKS[engine]) + for engine, nc_format in [ + ("netcdf4", "NETCDF3_CLASSIC"), + ("netcdf4", "NETCDF4_CLASSIC"), + ("netcdf4", "NETCDF4"), + ("h5netcdf", "NETCDF4"), + ("scipy", "NETCDF3_64BIT"), + ] ] @@ -93,9 +93,6 @@ def test_dask_distributed_netcdf_roundtrip( nc_format, compute, ): - if engine not in ENGINES: - pytest.skip("engine not available") - chunks = {"dim1": 4, "dim2": 3, "dim3": 6} with cluster() as (s, [_a, _b]): @@ -179,12 +176,19 @@ def test_open_mfdataset_multiple_files_parallel_distributed(parallel, tmp_path): # TODO: move this to test_backends.py @requires_cftime @requires_netCDF4 -@pytest.mark.parametrize("parallel", (True, False)) +@pytest.mark.parametrize( + "parallel", + ( + pytest.param( + True, + marks=pytest.mark.skip( + reason="Flaky in CI. Would be a welcome contribution to make a similar test reliable." + ), + ), + False, + ), +) def test_open_mfdataset_multiple_files_parallel(parallel, tmp_path): - if parallel: - pytest.skip( - "Flaky in CI. Would be a welcome contribution to make a similar test reliable." - ) lon = np.arange(100) time = xr.date_range("20010101", periods=100, calendar="360_day", use_cftime=True) data = np.random.random((time.size, lon.size)) @@ -211,9 +215,6 @@ def test_dask_distributed_read_netcdf_integration_test( engine, nc_format, ): - if engine not in ENGINES: - pytest.skip("engine not available") - chunks = {"dim1": 4, "dim2": 3, "dim3": 6} with cluster() as (s, [_a, _b]): @@ -234,7 +235,8 @@ def test_dask_distributed_read_netcdf_integration_test( # https://github.com/dask/dask/blob/e04734b4d8959ba259801f2e2a490cb4ee8d891f/dask/tests/test_distributed.py#L338-L358 @pytest.fixture def zarr(client): # noqa: F811 - zarr_lib = pytest.importorskip("zarr") + import zarr as zarr_lib + # Zarr-Python 3 lazily allocates a dedicated thread/IO loop # for to execute async tasks. To avoid having this thread # be picked up as a "leaked thread", we manually trigger it's diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 4ed66509725..01b18c29956 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -4,7 +4,7 @@ import pytest from xarray.core import dtypes -from xarray.tests import requires_array_api_strict +from xarray.tests import requires_array_api_strict, requires_dask try: import array_api_strict @@ -58,10 +58,11 @@ def test_result_type_scalars(values, expected) -> None: assert np.issubdtype(actual, expected) +@requires_dask def test_result_type_dask_array() -> None: # verify it works without evaluating dask arrays - da = pytest.importorskip("dask.array") - dask = pytest.importorskip("dask") + import dask + import dask.array as da def error(): raise RuntimeError diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 08fdc798130..9a98ef0cbe7 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -2,6 +2,7 @@ import copy import datetime as dt +import itertools import pickle import warnings from typing import Any @@ -42,13 +43,13 @@ dask_array_type, has_dask, has_dask_array_expr, - has_scipy, raise_if_dask_computes, requires_bottleneck, requires_cftime, requires_cupy, requires_dask, requires_pyarrow, + requires_scipy, ) @@ -439,15 +440,14 @@ def assert_dask_array(da, dask): @arm_xfail @pytest.mark.filterwarnings("ignore:All-NaN .* encountered:RuntimeWarning") -@pytest.mark.parametrize("dask", [False, True] if has_dask else [False]) -def test_datetime_mean(dask: bool, time_unit: PDDatetimeUnitOptions) -> None: +def test_datetime_mean(use_dask: bool, time_unit: PDDatetimeUnitOptions) -> None: # Note: only testing numpy, as dask is broken upstream dtype = f"M8[{time_unit}]" da = DataArray( np.array(["2010-01-01", "NaT", "2010-01-03", "NaT", "NaT"], dtype=dtype), dims=["time"], ) - if dask: + if use_dask: # Trigger use case where a chunk is full of NaT da = da.chunk({"time": 3}) @@ -455,12 +455,12 @@ def test_datetime_mean(dask: bool, time_unit: PDDatetimeUnitOptions) -> None: expect_nat = DataArray(np.array("NaT", dtype="M8[ns]")) actual = da.mean() - if dask: + if use_dask: assert actual.chunks is not None assert_equal(actual, expect) actual = da.mean(skipna=False) - if dask: + if use_dask: assert actual.chunks is not None assert_equal(actual, expect_nat) @@ -476,16 +476,12 @@ def test_datetime_mean(dask: bool, time_unit: PDDatetimeUnitOptions) -> None: @requires_cftime -@pytest.mark.parametrize("dask", [False, True]) -def test_cftime_datetime_mean(dask): - if dask and not has_dask: - pytest.skip("requires dask") - +def test_cftime_datetime_mean(use_dask): times = date_range("2000", periods=4, use_cftime=True) da = DataArray(times, dims=["time"]) da_2d = DataArray(times.values.reshape(2, 2)) - if dask: + if use_dask: da = da.chunk({"time": 2}) da_2d = da_2d.chunk({"dim_0": 2}) @@ -493,28 +489,25 @@ def test_cftime_datetime_mean(dask): # one compute needed to check the array contains cftime datetimes with raise_if_dask_computes(max_computes=1): result = da.isel(time=0).mean() - assert_dask_array(result, dask) + assert_dask_array(result, use_dask) assert_equal(result, expected) expected = DataArray(times.date_type(2000, 1, 2, 12)) with raise_if_dask_computes(max_computes=1): result = da.mean() - assert_dask_array(result, dask) + assert_dask_array(result, use_dask) assert_equal(result, expected) with raise_if_dask_computes(max_computes=1): result = da_2d.mean() - assert_dask_array(result, dask) + assert_dask_array(result, use_dask) assert_equal(result, expected) -@pytest.mark.parametrize("dask", [False, True]) -def test_mean_over_long_spanning_datetime64(dask) -> None: - if dask and not has_dask: - pytest.skip("requires dask") +def test_mean_over_long_spanning_datetime64(use_dask) -> None: array = np.array(["1678-01-01", "NaT", "2260-01-01"], dtype="datetime64[ns]") da = DataArray(array, dims=["time"]) - if dask: + if use_dask: da = da.chunk({"time": 2}) expected = DataArray(np.array("1969-01-01", dtype="datetime64[ns]")) result = da.mean() @@ -585,29 +578,28 @@ def test_empty_axis_dtype(): assert_identical(ds.sum(dim="time")["var"], ds["var"]) -@pytest.mark.parametrize("dim_num", [1, 2]) -@pytest.mark.parametrize("dtype", [float, int, np.float32, np.bool_]) -@pytest.mark.parametrize("dask", [False, True]) -@pytest.mark.parametrize("func", ["sum", "min", "max", "mean", "var"]) -# TODO test cumsum, cumprod -@pytest.mark.parametrize("skipna", [False, True]) -@pytest.mark.parametrize("aggdim", [None, "x"]) -def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): - if aggdim == "y" and dim_num < 2: - pytest.skip("dim not in this test") - - if dtype == np.bool_ and func == "mean": - pytest.skip("numpy does not support this") +def _reduce_params(): + for dtype, use_dask, func, skipna in itertools.product( + [float, int, np.float32, np.bool_], + [False, True], + ["sum", "min", "max", "mean", "var"], # TODO test cumsum, cumprod + [False, True], + ): + if use_dask and not skipna and dtype == np.bool_ and func in ("min", "max"): + # bool with NaN is object-typed, where min/max without skipna depends + # on the comparison order, which differs between dask and numpy. + continue + marks = [requires_dask] if use_dask else [] + yield pytest.param(dtype, use_dask, func, skipna, marks=marks) - if dask and not has_dask: - pytest.skip("requires dask") - - if dask and skipna is False and dtype == np.bool_: - pytest.skip("dask does not compute object-typed array") +@pytest.mark.parametrize("dim_num", [1, 2]) +@pytest.mark.parametrize("dtype, use_dask, func, skipna", list(_reduce_params())) +@pytest.mark.parametrize("aggdim", [None, "x"]) +def test_reduce(dim_num, dtype, use_dask, func, skipna, aggdim): rtol = 1e-04 if dtype == np.float32 else 1e-05 - da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask) + da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=use_dask) axis = None if aggdim is None else da.get_axis_num(aggdim) # TODO: remove these after resolving @@ -626,7 +618,7 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): expected = getattr(np, func)(da.values, axis=axis) actual = getattr(da, func)(skipna=skipna, dim=aggdim) - assert_dask_array(actual, dask) + assert_dask_array(actual, use_dask) np.testing.assert_allclose( actual.values, np.array(expected), rtol=1.0e-4, equal_nan=True ) @@ -647,7 +639,7 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): assert_allclose(actual, expected, rtol=rtol) # also check ddof!=0 case actual = getattr(da, func)(skipna=skipna, dim=aggdim, ddof=5) - if dask: + if use_dask: assert isinstance(da.data, dask_array_type) expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=5) assert_allclose(actual, expected, rtol=rtol) @@ -658,13 +650,13 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): # make sure the dtype argument if func not in ["max", "min"]: actual = getattr(da, func)(skipna=skipna, dim=aggdim, dtype=float) - assert_dask_array(actual, dask) + assert_dask_array(actual, use_dask) assert actual.dtype == float # without nan - da = construct_dataarray(dim_num, dtype, contains_nan=False, dask=dask) + da = construct_dataarray(dim_num, dtype, contains_nan=False, dask=use_dask) actual = getattr(da, func)(skipna=skipna) - if dask: + if use_dask: assert isinstance(da.data, dask_array_type) expected = getattr(np, f"nan{func}")(da.values) if actual.dtype == object: @@ -673,29 +665,24 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): assert np.allclose(actual.values, np.array(expected), rtol=rtol) -@pytest.mark.parametrize("dim_num", [1, 2]) -@pytest.mark.parametrize("dtype", [float, int, np.float32, np.bool_, str]) -@pytest.mark.parametrize("contains_nan", [True, False]) -@pytest.mark.parametrize("dask", [False, True]) +def _argmin_max_params(): + for dtype, contains_nan, skipna in itertools.product( + [float, int, np.float32, np.bool_, str], [True, False], [False, True] + ): + if contains_nan and not skipna and dtype in (np.bool_, str): + # bool and str with NaN are object-typed, where argmin/argmax without + # skipna either fails to compare NaN with str or depends on the order. + continue + yield (dtype, contains_nan, skipna) + + +@pytest.mark.parametrize("dim_num, aggdim", [(1, "x"), (2, "x"), (2, "y")]) +@pytest.mark.parametrize("dtype, contains_nan, skipna", list(_argmin_max_params())) @pytest.mark.parametrize("func", ["min", "max"]) -@pytest.mark.parametrize("skipna", [False, True]) -@pytest.mark.parametrize("aggdim", ["x", "y"]) -def test_argmin_max(dim_num, dtype, contains_nan, dask, func, skipna, aggdim): +def test_argmin_max(dim_num, dtype, contains_nan, use_dask, func, skipna, aggdim): # pandas-dev/pandas#16830, we do not check consistency with pandas but # just make sure da[da.argmin()] == da.min() - - if aggdim == "y" and dim_num < 2: - pytest.skip("dim not in this test") - - if dask and not has_dask: - pytest.skip("requires dask") - - if contains_nan: - if not skipna: - pytest.skip("numpy's argmin (not nanargmin) does not handle object-dtype") - if skipna and np.dtype(dtype).kind in "iufc": - pytest.skip("numpy's nanargmin raises ValueError for all nan axis") - da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, dask=dask) + da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, dask=use_dask) with warnings.catch_warnings(): warnings.filterwarnings("ignore", "All-NaN slice") @@ -835,16 +822,12 @@ def test_dask_gradient(axis, edge_order): @pytest.mark.parametrize("dim_num", [1, 2]) @pytest.mark.parametrize("dtype", [float, int, np.float32, np.bool_]) -@pytest.mark.parametrize("dask", [False, True]) @pytest.mark.parametrize("func", ["sum", "prod"]) @pytest.mark.parametrize("aggdim", [None, "x"]) @pytest.mark.parametrize("contains_nan", [True, False]) @pytest.mark.parametrize("skipna", [True, False, None]) -def test_min_count(dim_num, dtype, dask, func, aggdim, contains_nan, skipna): - if dask and not has_dask: - pytest.skip("requires dask") - - da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, dask=dask) +def test_min_count(dim_num, dtype, use_dask, func, aggdim, contains_nan, skipna): + da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, dask=use_dask) min_count = 3 # If using Dask, the function call should be lazy. @@ -853,19 +836,15 @@ def test_min_count(dim_num, dtype, dask, func, aggdim, contains_nan, skipna): expected = series_reduce(da, func, skipna=skipna, dim=aggdim, min_count=min_count) assert_allclose(actual, expected) - assert_dask_array(actual, dask) + assert_dask_array(actual, use_dask) @pytest.mark.parametrize("dtype", [float, int, np.float32, np.bool_]) -@pytest.mark.parametrize("dask", [False, True]) @pytest.mark.parametrize("func", ["sum", "prod"]) -def test_min_count_nd(dtype, dask, func): - if dask and not has_dask: - pytest.skip("requires dask") - +def test_min_count_nd(dtype, use_dask, func): min_count = 3 dim_num = 3 - da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask) + da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=use_dask) # If using Dask, the function call should be lazy. with raise_if_dask_computes(): @@ -877,23 +856,19 @@ def test_min_count_nd(dtype, dask, func): expected = getattr(da, func)(dim=..., skipna=True, min_count=min_count) assert_allclose(actual, expected) - assert_dask_array(actual, dask) + assert_dask_array(actual, use_dask) -@pytest.mark.parametrize("dask", [False, True]) @pytest.mark.parametrize("func", ["sum", "prod"]) @pytest.mark.parametrize("dim", [None, "a", "b"]) -def test_min_count_specific(dask, func, dim): - if dask and not has_dask: - pytest.skip("requires dask") - +def test_min_count_specific(use_dask, func, dim): # Simple array with four non-NaN values. da = DataArray(np.ones((6, 6), dtype=np.float64) * np.nan, dims=("a", "b")) da[0][0] = 2 da[0][3] = 2 da[3][0] = 2 da[3][3] = 2 - if dask: + if use_dask: da = da.chunk({"a": 3, "b": 3}) # Expected result if we set min_count to the number of non-NaNs in a @@ -910,7 +885,7 @@ def test_min_count_specific(dask, func, dim): # Check for that min_count. with raise_if_dask_computes(): actual = getattr(da, func)(dim, skipna=True, min_count=min_count) - assert_dask_array(actual, dask) + assert_dask_array(actual, use_dask) assert_allclose(actual, expected) # With min_count being one higher, should get all NaN. @@ -918,7 +893,7 @@ def test_min_count_specific(dask, func, dim): expected *= np.nan with raise_if_dask_computes(): actual = getattr(da, func)(dim, skipna=True, min_count=min_count) - assert_dask_array(actual, dask) + assert_dask_array(actual, use_dask) assert_allclose(actual, expected) @@ -932,26 +907,19 @@ def test_min_count_dataset(func): @pytest.mark.parametrize("dtype", [float, int, np.float32, np.bool_]) -@pytest.mark.parametrize("dask", [False, True]) @pytest.mark.parametrize("skipna", [False, True]) @pytest.mark.parametrize("func", ["sum", "prod"]) -def test_multiple_dims(dtype, dask, skipna, func): - if dask and not has_dask: - pytest.skip("requires dask") - da = construct_dataarray(3, dtype, contains_nan=True, dask=dask) +def test_multiple_dims(dtype, use_dask, skipna, func): + da = construct_dataarray(3, dtype, contains_nan=True, dask=use_dask) actual = getattr(da, func)(("x", "y"), skipna=skipna) expected = getattr(getattr(da, func)("x", skipna=skipna), func)("y", skipna=skipna) assert_allclose(actual, expected) -@pytest.mark.parametrize("dask", [True, False]) -def test_datetime_to_numeric_datetime64(dask, time_unit: PDDatetimeUnitOptions): - if dask and not has_dask: - pytest.skip("requires dask") - +def test_datetime_to_numeric_datetime64(use_dask, time_unit: PDDatetimeUnitOptions): times = pd.date_range("2000", periods=5, freq="7D").as_unit(time_unit).values - if dask: + if use_dask: times = dask_array_api.from_array(times, chunks=-1) with raise_if_dask_computes(): @@ -977,15 +945,11 @@ def test_datetime_to_numeric_datetime64(dask, time_unit: PDDatetimeUnitOptions): @requires_cftime -@pytest.mark.parametrize("dask", [True, False]) -def test_datetime_to_numeric_cftime(dask): - if dask and not has_dask: - pytest.skip("requires dask") - +def test_datetime_to_numeric_cftime(use_dask): times = date_range( "2000", periods=5, freq="7D", calendar="standard", use_cftime=True ).values - if dask: + if use_dask: times = dask_array_api.from_array(times, chunks=-1) with raise_if_dask_computes(): result = duck_array_ops.datetime_to_numeric(times, datetime_unit="h", dtype=int) @@ -1009,7 +973,7 @@ def test_datetime_to_numeric_cftime(dask): np.testing.assert_array_equal(result, expected2) with raise_if_dask_computes(): - if dask: + if use_dask: time = dask_array_api.asarray(times[1]) else: time = np.asarray(times[1]) @@ -1021,11 +985,11 @@ def test_datetime_to_numeric_cftime(dask): @requires_cftime +# "ns" is excluded: out-of-bounds datetime64 overflow +@pytest.mark.parametrize("time_unit", ["s", "ms", "us"]) def test_datetime_to_numeric_potential_overflow(time_unit: PDDatetimeUnitOptions): import cftime - if time_unit == "ns": - pytest.skip("out-of-bounds datetime64 overflow") dtype = f"M8[{time_unit}]" times = pd.date_range("2000", periods=5, freq="7D").values.astype(dtype) cftimes = date_range( @@ -1100,11 +1064,12 @@ def test_timedelta_to_numeric(td, time_unit: PDDatetimeUnitOptions): assert isinstance(out, float) -@pytest.mark.parametrize("use_dask", [True, False]) +@pytest.mark.parametrize( + "use_dask", + [pytest.param(True, marks=[requires_dask, requires_scipy]), False], +) @pytest.mark.parametrize("skipna", [True, False]) def test_least_squares(use_dask, skipna): - if use_dask and (not has_dask or not has_scipy): - pytest.skip("requires dask and scipy") lhs = np.array([[1, 2], [1, 2], [3, 2]]) rhs = DataArray(np.array([3, 5, 7]), dims=("y",)) diff --git a/xarray/tests/test_duck_array_wrapping.py b/xarray/tests/test_duck_array_wrapping.py index a3089364055..898bf9bd62e 100644 --- a/xarray/tests/test_duck_array_wrapping.py +++ b/xarray/tests/test_duck_array_wrapping.py @@ -1,9 +1,19 @@ +import importlib + import numpy as np import pandas as pd import pytest import xarray as xr -from xarray.tests import get_dask_chunkmanager, has_dask_array_expr +from xarray.tests import ( + get_dask_chunkmanager, + has_dask_array_expr, + requires_cupy, + requires_dask, + requires_jax, + requires_pint, + requires_sparse, +) # Don't run cupy in CI because it requires a GPU NAMESPACE_ARRAYS = { @@ -102,6 +112,18 @@ }, } +NAMESPACE_REQUIREMENTS = { + "cupy": requires_cupy, + "dask.array": requires_dask, + "jax.numpy": requires_jax, + "pint": requires_pint, + "sparse": requires_sparse, +} +NAMESPACES = [ + pytest.param(namespace, marks=NAMESPACE_REQUIREMENTS[namespace]) + for namespace in NAMESPACE_ARRAYS +] + try: import jax # type: ignore[import-not-found,unused-ignore] @@ -127,7 +149,7 @@ def constructor(data): self.constructor = constructor else: - self.xp = pytest.importorskip(namespace) + self.xp = importlib.import_module(namespace) self.Array = getattr(self.xp, NAMESPACE_ARRAYS[namespace]["attrs"]["array"]) self.constructor = getattr( self.xp, NAMESPACE_ARRAYS[namespace]["attrs"]["constructor"] @@ -148,7 +170,7 @@ def get_test_dataarray(self): ) -@pytest.mark.parametrize("namespace", NAMESPACE_ARRAYS) +@pytest.mark.parametrize("namespace", NAMESPACES) class TestTopLevelMethods(_BaseTest): @pytest.fixture(autouse=True) def setUp(self, request, namespace): @@ -210,7 +232,7 @@ def test_map_blocks(self): assert isinstance(result.data, self.Array) -@pytest.mark.parametrize("namespace", NAMESPACE_ARRAYS) +@pytest.mark.parametrize("namespace", NAMESPACES) class TestDataArrayMethods(_BaseTest): @pytest.fixture(autouse=True) def setUp(self, request, namespace): diff --git a/xarray/tests/test_formatting_html.py b/xarray/tests/test_formatting_html.py index 5160cc29066..aefd8b82722 100644 --- a/xarray/tests/test_formatting_html.py +++ b/xarray/tests/test_formatting_html.py @@ -10,6 +10,7 @@ import xarray as xr from xarray.core import formatting_html as fh from xarray.core.coordinates import Coordinates +from xarray.tests import requires_dask def drop_fallback_text_repr(html: str) -> str: @@ -79,7 +80,6 @@ def dataarray() -> xr.DataArray: @pytest.fixture def dask_dataarray(dataarray: xr.DataArray) -> xr.DataArray: - pytest.importorskip("dask") return dataarray.chunk() @@ -121,6 +121,7 @@ def test_short_data_repr_html_non_str_keys(dataset: xr.Dataset) -> None: fh.dataset_repr(ds) +@requires_dask def test_short_data_repr_html_dask(dask_dataarray: xr.DataArray) -> None: assert hasattr(dask_dataarray.data, "_repr_html_") data_repr = fh.short_data_repr_html(dask_dataarray) diff --git a/xarray/tests/test_groupby.py b/xarray/tests/test_groupby.py index b00ef4da98c..34db8c9dcb0 100644 --- a/xarray/tests/test_groupby.py +++ b/xarray/tests/test_groupby.py @@ -41,9 +41,7 @@ create_test_data, dask_array_api, has_cftime, - has_dask, has_dask_array_expr, - has_dask_ge_2024_08_1, has_flox, raise_if_dask_computes, requires_cftime, @@ -683,20 +681,14 @@ def test_groupby_repr_datetime(obj) -> None: @pytest.mark.parametrize( "chunk", [ - pytest.param( - dict(lat=1), marks=pytest.mark.skipif(not has_dask, reason="no dask") - ), - pytest.param( - dict(lat=2, lon=2), marks=pytest.mark.skipif(not has_dask, reason="no dask") - ), + pytest.param(dict(lat=1), marks=requires_dask), + pytest.param(dict(lat=2, lon=2), marks=requires_dask), False, ], ) def test_groupby_drops_nans(shuffle: bool, chunk: Literal[False] | dict) -> None: if chunk and has_dask_array_expr: pytest.xfail("flox groupby currently builds legacy dask arrays") - if shuffle and chunk and not has_dask_ge_2024_08_1: - pytest.skip() # GH2383 # nan in 2D data variable (requires stacking) ds = xr.Dataset( @@ -1445,24 +1437,12 @@ def test_groupby_sum(self) -> None: @pytest.mark.parametrize("use_flox", [True, False]) @pytest.mark.parametrize("shuffle", [True, False]) - @pytest.mark.parametrize( - "chunk", - [ - pytest.param( - True, marks=pytest.mark.skipif(not has_dask, reason="no dask") - ), - False, - ], - ) @pytest.mark.parametrize("method", ["sum", "mean", "median"]) def test_groupby_reductions( - self, use_flox: bool, method: str, shuffle: bool, chunk: bool + self, use_flox: bool, method: str, shuffle: bool, use_dask: bool ) -> None: - if shuffle and chunk and not has_dask_ge_2024_08_1: - pytest.skip() - array = self.da - if chunk: + if use_dask: array.data = array.chunk({"y": 5}).data reduction = getattr(np, method) expected = Dataset( @@ -1944,8 +1924,6 @@ class TestDataArrayResample: def test_resample( self, use_cftime: bool, shuffle: bool, resample_freq: ResampleCompatible ) -> None: - if use_cftime and not has_cftime: - pytest.skip() times = xr.date_range( "2000-01-01", freq="6h", periods=10, use_cftime=use_cftime ) @@ -2417,8 +2395,6 @@ class TestDatasetResample: def test_resample( self, use_cftime: bool, resample_freq: ResampleCompatible ) -> None: - if use_cftime and not has_cftime: - pytest.skip() times = xr.date_range( "2000-01-01", freq="6h", periods=10, use_cftime=use_cftime ) @@ -2630,85 +2606,113 @@ def test_resample_ignores_keepdims(self, use_flox: bool, func) -> None: getattr(ds.resample(time="MS"), func)(keepdims=True) -@pytest.mark.parametrize("use_lazy_group_idx", [True, False]) -@pytest.mark.parametrize("use_dask", [True, False]) -@pytest.mark.parametrize("use_flox", [True, False]) +_SCAN_CASES = [ + ( + "cumsum", + ["group_idx"], + "time", + [[7, 9, 0, 1, 2, 2], [1, 2, 1, 2, 1, 2], [2, 4, 2, 4, 2, 4]], + ), + ( + "cumsum", + ["group_idx"], + "test", + [[7, 2, 0, 1, 2, 0], [8, 3, 1, 2, 3, 1], [10, 5, 3, 4, 5, 3]], + ), + ( + "cumsum", + ["group_idx"], + ..., + [[7, 9, 0, 1, 2, 2], [8, 11, 1, 3, 3, 4], [10, 15, 3, 7, 5, 8]], + ), + ( + "cumsum", + ["group_idx", "group_idx2"], + "time", + [[7, 2, 0, 1, 2, 2], [1, 1, 1, 2, 1, 2], [2, 2, 2, 4, 2, 4]], + ), + ( + "cumsum", + ["group_idx", "group_idx2"], + "test", + [[7, 2, 0, 1, 2, 0], [8, 3, 1, 2, 3, 1], [10, 5, 3, 4, 5, 3]], + ), + ( + "cumsum", + ["group_idx", "group_idx2"], + ..., + [[7, 2, 0, 1, 2, 2], [8, 3, 1, 3, 3, 4], [10, 5, 3, 7, 5, 8]], + ), + ( + "cumprod", + ["group_idx"], + "time", + [[7, 14, 0, 0, 2, 2], [1, 1, 1, 1, 1, 1], [2, 4, 2, 4, 2, 4]], + ), + ( + "cumprod", + ["group_idx"], + "test", + [[7, 2, 0, 1, 2, 1], [7, 2, 0, 1, 2, 1], [14, 4, 0, 2, 4, 2]], + ), + ( + "cumprod", + ["group_idx"], + ..., + [[7, 14, 0, 0, 2, 2], [7, 14, 0, 0, 2, 2], [14, 56, 0, 0, 4, 8]], + ), + ( + "cumprod", + ["group_idx", "group_idx2"], + "time", + [[7, 2, 0, 0, 2, 2], [1, 1, 1, 1, 1, 1], [2, 2, 2, 4, 2, 4]], + ), + ( + "cumprod", + ["group_idx", "group_idx2"], + "test", + [[7, 2, 0, 1, 2, 1], [7, 2, 0, 1, 2, 1], [14, 4, 0, 2, 4, 2]], + ), + ( + "cumprod", + ["group_idx", "group_idx2"], + ..., + [[7, 2, 0, 0, 2, 2], [7, 2, 0, 0, 2, 2], [14, 4, 0, 0, 4, 8]], + ), +] + + +def _groupby_scans_params(): + for method, grp_idx, dim, expected_array in _SCAN_CASES: + for use_flox in (True, False): + for use_lazy_group_idx in (True, False): + if use_lazy_group_idx and not use_flox: + # Lazy group_idx is not supported without flox. + continue + marks = [requires_flox] if use_flox else [] + if use_flox and method == "cumprod": + reason = "TODO: Groupby with cumprod is currently not supported with flox" + marks.append(pytest.mark.skip(reason=reason)) + elif use_flox and dim == ...: + reason = "TODO: Scans are only supported along a single dimension in flox." + marks.append(pytest.mark.skip(reason=reason)) + elif use_flox and dim == "test": + reason = "TODO: group_idx along time dim and axis along test dim not currently supported with flox." + marks.append(pytest.mark.skip(reason=reason)) + yield pytest.param( + method, + grp_idx, + dim, + expected_array, + use_flox, + use_lazy_group_idx, + marks=marks, + ) + + @pytest.mark.parametrize( - "method, grp_idx, dim, expected_array", - [ - ( - "cumsum", - ["group_idx"], - "time", - [[7, 9, 0, 1, 2, 2], [1, 2, 1, 2, 1, 2], [2, 4, 2, 4, 2, 4]], - ), - ( - "cumsum", - ["group_idx"], - "test", - [[7, 2, 0, 1, 2, 0], [8, 3, 1, 2, 3, 1], [10, 5, 3, 4, 5, 3]], - ), - ( - "cumsum", - ["group_idx"], - ..., - [[7, 9, 0, 1, 2, 2], [8, 11, 1, 3, 3, 4], [10, 15, 3, 7, 5, 8]], - ), - ( - "cumsum", - ["group_idx", "group_idx2"], - "time", - [[7, 2, 0, 1, 2, 2], [1, 1, 1, 2, 1, 2], [2, 2, 2, 4, 2, 4]], - ), - ( - "cumsum", - ["group_idx", "group_idx2"], - "test", - [[7, 2, 0, 1, 2, 0], [8, 3, 1, 2, 3, 1], [10, 5, 3, 4, 5, 3]], - ), - ( - "cumsum", - ["group_idx", "group_idx2"], - ..., - [[7, 2, 0, 1, 2, 2], [8, 3, 1, 3, 3, 4], [10, 5, 3, 7, 5, 8]], - ), - ( - "cumprod", - ["group_idx"], - "time", - [[7, 14, 0, 0, 2, 2], [1, 1, 1, 1, 1, 1], [2, 4, 2, 4, 2, 4]], - ), - ( - "cumprod", - ["group_idx"], - "test", - [[7, 2, 0, 1, 2, 1], [7, 2, 0, 1, 2, 1], [14, 4, 0, 2, 4, 2]], - ), - ( - "cumprod", - ["group_idx"], - ..., - [[7, 14, 0, 0, 2, 2], [7, 14, 0, 0, 2, 2], [14, 56, 0, 0, 4, 8]], - ), - ( - "cumprod", - ["group_idx", "group_idx2"], - "time", - [[7, 2, 0, 0, 2, 2], [1, 1, 1, 1, 1, 1], [2, 2, 2, 4, 2, 4]], - ), - ( - "cumprod", - ["group_idx", "group_idx2"], - "test", - [[7, 2, 0, 1, 2, 1], [7, 2, 0, 1, 2, 1], [14, 4, 0, 2, 4, 2]], - ), - ( - "cumprod", - ["group_idx", "group_idx2"], - ..., - [[7, 2, 0, 0, 2, 2], [7, 2, 0, 0, 2, 2], [14, 4, 0, 0, 4, 8]], - ), - ], + "method, grp_idx, dim, expected_array, use_flox, use_lazy_group_idx", + list(_groupby_scans_params()), ) def test_groupby_scans( method: Literal["cumsum", "cumprod"], @@ -2719,31 +2723,9 @@ def test_groupby_scans( use_dask: bool, use_lazy_group_idx: bool, ) -> None: - if use_dask and not has_dask: - pytest.skip("requires dask") - if use_dask and use_flox and has_dask_array_expr: pytest.xfail("flox groupby scans currently mix legacy dask arrays") - if use_flox: - if not has_flox: - pytest.skip("requires flox") - - if method == "cumprod": - pytest.skip( - "TODO: Groupby with cumprod is currently not supported with flox" - ) - if dim == ...: - pytest.skip( - "TODO: Scans are only supported along a single dimension in flox." - ) - elif dim == "test": - pytest.skip( - "TODO: group_idx along time dim and axis along test dim not currently supported with flox." - ) - elif use_lazy_group_idx: - pytest.skip("Lazy group_idx is not supported without flox.") - # Test Dataset groupby: ds = xr.Dataset( { @@ -2878,10 +2860,8 @@ def test_min_count_vs_flox(func: str, min_count: int | None, skipna: bool) -> No assert_identical(actual, expected) -@pytest.mark.parametrize("use_flox", [True, False]) +@pytest.mark.parametrize("use_flox", [pytest.param(True, marks=requires_flox), False]) def test_min_count_error(use_flox: bool) -> None: - if use_flox and not has_flox: - pytest.skip() da = DataArray( data=np.array([np.nan, 1, 1, np.nan, 1, 1]), dims="x", @@ -3155,28 +3135,6 @@ def test_multiple_groupers(use_flox: bool, shuffle: bool) -> None: # TODO: is order of dims correct? assert_identical(actual, expected.transpose("z", "x", "xy")) - if has_dask: - b["xy"] = b["xy"].chunk() - expected = xr.DataArray( - [[[1, 1, 1], [np.nan, 1, 2]]] * 4, - dims=("z", "x", "xy"), - coords={"xy": ("xy", ["a", "b", "c"], {"foo": "bar"})}, - ) - with raise_if_dask_computes(max_computes=0): - gb = b.groupby(x=UniqueGrouper(), xy=UniqueGrouper(labels=["a", "b", "c"])) - assert is_chunked_array(gb.encoded.codes.data) - assert not gb.encoded.group_indices - if has_flox: - if has_dask_array_expr: - pytest.xfail( - "flox lazy multiple-groupers currently mix legacy dask arrays" - ) - with raise_if_dask_computes(max_computes=1): - assert_identical(gb.count(), expected) - else: - with pytest.raises(ValueError, match="when lazily grouping"): - gb.count() - @pytest.mark.parametrize("use_flox", [True, False]) @pytest.mark.parametrize("shuffle", [True, False]) @@ -3224,6 +3182,48 @@ def test_multiple_groupers_mixed(use_flox: bool, shuffle: bool) -> None: # ------ +@requires_dask +@pytest.mark.parametrize( + "use_flox", + [ + pytest.param( + True, + marks=[ + requires_flox, + pytest.mark.xfail( + has_dask_array_expr, + reason="flox lazy multiple-groupers currently mix legacy dask arrays", + ), + ], + ), + False, + ], +) +def test_multiple_groupers_lazy(use_flox: bool) -> None: + b = xr.DataArray( + np.random.default_rng(0).random((2, 3, 4)), + coords={"xy": (("x", "y"), [["a", "b", "c"], ["b", "c", "c"]], {"foo": "bar"})}, + dims=["x", "y", "z"], + ) + b["xy"] = b["xy"].chunk() + expected = xr.DataArray( + [[[1, 1, 1], [np.nan, 1, 2]]] * 4, + dims=("z", "x", "xy"), + coords={"xy": ("xy", ["a", "b", "c"], {"foo": "bar"})}, + ) + with raise_if_dask_computes(max_computes=0): + gb = b.groupby(x=UniqueGrouper(), xy=UniqueGrouper(labels=["a", "b", "c"])) + assert is_chunked_array(gb.encoded.codes.data) + assert not gb.encoded.group_indices + with xr.set_options(use_flox=use_flox): + if use_flox: + with raise_if_dask_computes(max_computes=1): + assert_identical(gb.count(), expected) + else: + with pytest.raises(ValueError, match="when lazily grouping"): + gb.count() + + @requires_flox_0_9_12 @pytest.mark.parametrize( "reduction", ["max", "min", "nanmax", "nanmin", "sum", "nansum", "prod", "nanprod"] @@ -3294,6 +3294,19 @@ def test_groupby_transpose() -> None: assert_identical(first, second.transpose(*first.dims)) +def test_groupby_non_leading_dim_preserves_coords() -> None: + data = xr.DataArray( + np.empty((0, 2)), + dims=["x", "y"], + coords={"x": [], "y": [1, 1]}, + ) + + actual = data.groupby("y").sum() + + assert "x" in actual.coords + assert actual.sizes == {"x": 0, "y": 1} + + @requires_dask @pytest.mark.parametrize( "grouper, expect_index", @@ -3718,9 +3731,6 @@ def test_season_resampling_raises_unsorted_seasons(self, seasons): with pytest.raises(ValueError, match="sort"): da.resample(time=SeasonResampler(seasons)) - @pytest.mark.parametrize( - "use_cftime", [pytest.param(True, marks=requires_cftime), False] - ) @pytest.mark.parametrize("drop_incomplete", [True, False]) @pytest.mark.parametrize( "seasons", @@ -3841,16 +3851,7 @@ def test_season_resampler_preserves_time_unit( assert result_unit == time_unit -@pytest.mark.parametrize( - "chunk", - [ - pytest.param( - True, marks=pytest.mark.skipif(not has_dask, reason="requires dask") - ), - False, - ], -) -def test_datetime_mean(chunk, use_cftime): +def test_datetime_mean(use_dask, use_cftime): ds = xr.Dataset( { "var1": ( @@ -3862,7 +3863,7 @@ def test_datetime_mean(chunk, use_cftime): "var2": (("x",), list(range(10))), } ) - if chunk: + if use_dask: ds = ds.chunk() assert "var1" in ds.groupby("x").mean("time") assert "var1" in ds.mean("x") diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py index 085579f8ee2..b42ea205810 100644 --- a/xarray/tests/test_interp.py +++ b/xarray/tests/test_interp.py @@ -21,13 +21,11 @@ assert_equal, assert_identical, dask_array_api, - has_dask, - has_scipy, - has_scipy_ge_1_13, raise_if_dask_computes, requires_cftime, requires_dask, requires_scipy, + requires_scipy_ge_1_13, ) from xarray.tests.test_dataset import create_test_data @@ -108,26 +106,23 @@ def nd_interp_coords(): return coords +@requires_scipy def test_keywargs(): - if not has_scipy: - pytest.skip("scipy is not installed.") - da = get_example_data(0) assert_equal(da.interp(x=[0.5, 0.8]), da.interp({"x": [0.5, 0.8]})) +@requires_scipy @pytest.mark.parametrize("method", ["linear", "cubic"]) @pytest.mark.parametrize("dim", ["x", "y"]) @pytest.mark.parametrize( - "case", [pytest.param(0, id="no_chunk"), pytest.param(1, id="chunk_y")] + "case", + [ + pytest.param(0, id="no_chunk"), + pytest.param(1, id="chunk_y", marks=requires_dask), + ], ) def test_interpolate_1d(method: InterpOptions, dim: str, case: int) -> None: - if not has_scipy: - pytest.skip("scipy is not installed.") - - if not has_dask and case == 1: - pytest.skip("dask is not installed in the environment.") - da = get_example_data(case) xdest = np.linspace(0.0, 0.9, 80) actual = da.interp(method=method, coords={dim: xdest}) @@ -152,11 +147,9 @@ def func(obj, new_x): assert_allclose(actual, expected) +@requires_scipy @pytest.mark.parametrize("method", ["cubic", "zero"]) def test_interpolate_1d_methods(method: InterpOptions) -> None: - if not has_scipy: - pytest.skip("scipy is not installed.") - da = get_example_data(0) dim = "x" xdest = np.linspace(0.0, 0.9, 80) @@ -188,17 +181,17 @@ def func(obj, new_x): pytest.param( False, "makima", - marks=pytest.mark.skipif(not has_scipy_ge_1_13, reason="scipy too old"), + marks=requires_scipy_ge_1_13, ), pytest.param( True, "linear", - marks=pytest.mark.skipif(not has_dask, reason="dask not available"), + marks=requires_dask, ), pytest.param( True, "akima", - marks=pytest.mark.skipif(not has_dask, reason="dask not available"), + marks=requires_dask, ), ), ) @@ -297,9 +290,7 @@ def func(obj, dim, new_x, method): "case", [ pytest.param(3, id="no_chunk"), - pytest.param( - 4, id="chunked", marks=pytest.mark.skipif(not has_dask, reason="no dask") - ), + pytest.param(4, id="chunked", marks=requires_dask), ], ) def test_interpolate_nd(case: int, method: InterpnOptions, nd_interp_coords) -> None: @@ -431,12 +422,13 @@ def test_interpolate_nd_with_nan() -> None: @requires_scipy @pytest.mark.parametrize("method", ("linear",)) @pytest.mark.parametrize( - "case", [pytest.param(0, id="no_chunk"), pytest.param(1, id="chunk_y")] + "case", + [ + pytest.param(0, id="no_chunk"), + pytest.param(1, id="chunk_y", marks=requires_dask), + ], ) def test_interpolate_scalar(method: InterpOptions, case: int) -> None: - if not has_dask and case == 1: - pytest.skip("dask is not installed in the environment.") - da = get_example_data(case) xdest = 0.4 @@ -461,12 +453,13 @@ def func(obj, new_x): @requires_scipy @pytest.mark.parametrize("method", ("linear",)) @pytest.mark.parametrize( - "case", [pytest.param(3, id="no_chunk"), pytest.param(4, id="chunked")] + "case", + [ + pytest.param(3, id="no_chunk"), + pytest.param(4, id="chunked", marks=requires_dask), + ], ) def test_interpolate_nd_scalar(method: InterpOptions, case: int) -> None: - if not has_dask and case == 4: - pytest.skip("dask is not installed in the environment.") - da = get_example_data(case) xdest = 0.4 ydest = 0.05 @@ -492,15 +485,11 @@ def test_interpolate_nd_scalar(method: InterpOptions, case: int) -> None: assert_allclose(actual, expected) -@pytest.mark.parametrize("use_dask", [True, False]) +@requires_scipy def test_nans(use_dask: bool) -> None: - if not has_scipy: - pytest.skip("scipy is not installed.") - da = xr.DataArray([0, 1, np.nan, 2], dims="x", coords={"x": range(4)}) - if not has_dask and use_dask: - pytest.skip("dask is not installed in the environment.") + if use_dask: da = da.chunk() actual = da.interp(x=[0.5, 1.5]) @@ -509,12 +498,10 @@ def test_nans(use_dask: bool) -> None: @requires_scipy -@pytest.mark.parametrize("use_dask", [True, False]) def test_errors(use_dask: bool) -> None: # spline is unavailable da = xr.DataArray([0, 1, np.nan, 2], dims="x", coords={"x": range(4)}) - if not has_dask and use_dask: - pytest.skip("dask is not installed in the environment.") + if use_dask: da = da.chunk() for method in ["spline"]: @@ -627,11 +614,10 @@ def test_dataset() -> None: assert interpolated["var1"].attrs["buz"] == "var2" +@requires_scipy @pytest.mark.parametrize("case", [pytest.param(0, id="2D"), pytest.param(3, id="3D")]) def test_interpolate_dimorder(case: int) -> None: """Make sure the resultant dimension order is consistent with .sel()""" - if not has_scipy: - pytest.skip("scipy is not installed.") da = get_example_data(case) @@ -889,15 +875,17 @@ def test_decompose(method: InterpOptions) -> None: @requires_scipy @requires_dask -@pytest.mark.parametrize("method", ("linear", "nearest", "cubic", "pchip", "quintic")) @pytest.mark.parametrize("chunked", [True, False]) @pytest.mark.parametrize( - "data_ndim,interp_ndim,nscalar", + "method,data_ndim,interp_ndim,nscalar", [ - (data_ndim, interp_ndim, nscalar) + (method, data_ndim, interp_ndim, nscalar) + for method in ("linear", "nearest", "cubic", "pchip", "quintic") for data_ndim in range(1, 4) for interp_ndim in range(1, data_ndim + 1) for nscalar in range(interp_ndim + 1) + # 3d interpolation with the higher order methods is too slow + if not (method in ("cubic", "pchip", "quintic") and interp_ndim == 3) ], ) @pytest.mark.filterwarnings("ignore:Increasing number of chunks") @@ -909,9 +897,6 @@ def test_interpolate_chunk_1d( It should do a series of 1d interpolation """ - if method in ["cubic", "pchip", "quintic"] and interp_ndim == 3: - pytest.skip("Too slow.") - # 3d non chunked data x = np.linspace(0, 1, 6) y = np.linspace(2, 4, 7) @@ -1161,16 +1146,7 @@ def test_interp_vectorized_dask() -> None: @requires_scipy -@pytest.mark.parametrize( - "chunk", - [ - pytest.param( - True, marks=pytest.mark.skipif(not has_dask, reason="requires_dask") - ), - False, - ], -) -def test_interp_vectorized_shared_dims(chunk: bool) -> None: +def test_interp_vectorized_shared_dims(use_dask: bool) -> None: # GH4463 da = xr.DataArray( [[[1, 2, 3], [2, 3, 4]], [[1, 2, 3], [2, 3, 4]]], @@ -1181,7 +1157,7 @@ def test_interp_vectorized_shared_dims(chunk: bool) -> None: dx = xr.DataArray( [[1.5, 1.5], [1.5, 1.5]], dims=("t", "u"), coords={"u": [45, 55], "t": [10, 12]} ) - if chunk: + if use_dask: da = da.chunk(t=1) with raise_if_dask_computes(): actual = da.interp(y=dy, x=dx, method="linear") diff --git a/xarray/tests/test_missing.py b/xarray/tests/test_missing.py index 9c2d6b47727..2ccb93d264f 100644 --- a/xarray/tests/test_missing.py +++ b/xarray/tests/test_missing.py @@ -678,10 +678,6 @@ def test_interpolate_na_max_gap_errors(da_time): @requires_bottleneck -@pytest.mark.parametrize( - "use_cftime", - [False, pytest.param(True, marks=requires_cftime)], -) @pytest.mark.parametrize("transform", [lambda x: x, lambda x: x.to_dataset(name="a")]) @pytest.mark.parametrize( "max_gap", ["3h", np.timedelta64(3, "h"), pd.to_timedelta("3h")] diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 09ff1795ef4..a1cd312a678 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -19,7 +19,7 @@ ) from xarray.namedarray.core import NamedArray, from_array from xarray.namedarray.utils import fake_target_chunksize -from xarray.tests import requires_cftime +from xarray.tests import requires_array_api_strict, requires_cftime if TYPE_CHECKING: from types import ModuleType @@ -383,13 +383,16 @@ def test_duck_array_class(self) -> None: custom_a = CustomArrayIndexable(numpy_a) check_duck_array_typevar(custom_a) + @requires_array_api_strict def test_duck_array_class_array_api(self) -> None: # Test numpy's array api: - nxp = pytest.importorskip("array_api_strict", minversion="1.0") + import array_api_strict as nxp # TODO: nxp doesn't use dtype typevars, so can only use Any for the moment: - arrayapi_a: duckarray[Any, Any] # duckarray[Any, np.dtype[np.int64]] - arrayapi_a = nxp.asarray([2.1, 4], dtype=nxp.int64) + arrayapi_a = cast( + "duckarray[Any, Any]", # duckarray[Any, np.dtype[np.int64]] + nxp.asarray([2.1, 4], dtype=nxp.int64), + ) check_duck_array_typevar(arrayapi_a) def test_new_namedarray(self) -> None: diff --git a/xarray/tests/test_pandas_to_xarray.py b/xarray/tests/test_pandas_to_xarray.py index 8346f5abe21..7c0b81de116 100644 --- a/xarray/tests/test_pandas_to_xarray.py +++ b/xarray/tests/test_pandas_to_xarray.py @@ -95,11 +95,12 @@ } -@pytest.fixture( - params=[ - key for key, value in indices_dict.items() if not isinstance(value, MultiIndex) - ] -) +flat_index_keys = [ + key for key, value in indices_dict.items() if not isinstance(value, MultiIndex) +] + + +@pytest.fixture(params=flat_index_keys) def index_flat(request): """ index fixture, but excluding MultiIndex cases. @@ -124,11 +125,15 @@ def df(self): } ) + # empty index is excluded: the test doesn't make sense for it + @pytest.mark.parametrize( + "index_flat", + [key for key in flat_index_keys if len(indices_dict[key]) > 0], + indirect=True, + ) def test_to_xarray_index_types(self, index_flat, df): index = index_flat # MultiIndex is tested in test_to_xarray_with_multiindex - if len(index) == 0: - pytest.skip("Test doesn't make sense for empty index") from xarray import Dataset diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 87ca6610e42..70a8ff5b027 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1345,8 +1345,6 @@ def test_complex_raises_typeerror(self) -> None: def test_3d_raises_valueerror(self) -> None: a = DataArray(easy_array((2, 3, 4))) - if self.plotfunc.__name__ == "imshow": - pytest.skip() with pytest.raises(ValueError, match=r"DataArray must be 2d"): self.plotfunc(a) @@ -2006,6 +2004,10 @@ def test_interval_breaks_logspace(self) -> None: class TestImshow(Common2dMixin, PlotTestCase): plotfunc = staticmethod(xplt.imshow) + @pytest.mark.skip(reason="imshow accepts 3d arrays as RGB(A) images") + def test_3d_raises_valueerror(self) -> None: + pass + @pytest.mark.xfail( reason=( "Failing inside matplotlib. Should probably be fixed upstream because " @@ -2204,32 +2206,32 @@ def test_2d_coord_names(self) -> None: assert "y2d" == ax.get_ylabel() assert f"{self.darray.long_name} [{self.darray.units}]" == ax.get_zlabel() + @pytest.mark.skip(reason="does not make sense for surface plots") def test_xyincrease_false_changes_axes(self) -> None: - # Does not make sense for surface plots - pytest.skip("does not make sense for surface plots") + pass + @pytest.mark.skip(reason="does not make sense for surface plots") def test_xyincrease_true_changes_axes(self) -> None: - # Does not make sense for surface plots - pytest.skip("does not make sense for surface plots") + pass def test_can_pass_in_axis(self) -> None: self.pass_in_axis(self.plotmethod, subplot_kw={"projection": "3d"}) + @pytest.mark.skip(reason="does not make sense for surface plots") def test_default_cmap(self) -> None: - # Does not make sense for surface plots with default arguments - pytest.skip("does not make sense for surface plots") + pass + @pytest.mark.skip(reason="does not make sense for surface plots") def test_diverging_color_limits(self) -> None: - # Does not make sense for surface plots with default arguments - pytest.skip("does not make sense for surface plots") + pass + @pytest.mark.skip(reason="does not make sense for surface plots") def test_colorbar_kwargs(self) -> None: - # Does not make sense for surface plots with default arguments - pytest.skip("does not make sense for surface plots") + pass + @pytest.mark.skip(reason="does not make sense for surface plots") def test_cmap_and_color_both(self) -> None: - # Does not make sense for surface plots with default arguments - pytest.skip("does not make sense for surface plots") + pass def test_seaborn_palette_as_cmap(self) -> None: # seaborn does not work with mpl_toolkits.mplot3d diff --git a/xarray/tests/test_print_versions.py b/xarray/tests/test_print_versions.py index f964eb88948..23b6350fb57 100644 --- a/xarray/tests/test_print_versions.py +++ b/xarray/tests/test_print_versions.py @@ -2,9 +2,13 @@ import io +import pytest + import xarray +# importing setuptools after distutils (done by some dependencies) warns +@pytest.mark.filterwarnings("ignore:Setuptools is replacing distutils:UserWarning") def test_show_versions() -> None: f = io.StringIO() xarray.show_versions(file=f) diff --git a/xarray/tests/test_range_index.py b/xarray/tests/test_range_index.py index 1df14170007..a06e04cb502 100644 --- a/xarray/tests/test_range_index.py +++ b/xarray/tests/test_range_index.py @@ -39,6 +39,19 @@ def test_range_index_arange(args, kwargs) -> None: assert_equal(actual, expected, check_default_indexes=False) +@pytest.mark.parametrize( + ("start", "stop", "step"), + [(0.0, 3.0, -1.0), (3.0, 0.0, 1.0)], +) +def test_range_index_arange_empty(start, stop, step) -> None: + index = RangeIndex.arange(start, stop, step, dim="x") + coords = xr.Coordinates.from_xindex(index) + + assert index.size == np.arange(start, stop, step).size == 0 + assert coords.sizes == {"x": 0} + assert xr.DataArray([], dims="x", coords=coords).sizes == {"x": 0} + + def test_range_index_arange_error() -> None: with pytest.raises(TypeError, match=r".*requires stop to be specified"): RangeIndex.arange(dim="x") diff --git a/xarray/tests/test_rolling.py b/xarray/tests/test_rolling.py index aa513366171..71b5269b14e 100644 --- a/xarray/tests/test_rolling.py +++ b/xarray/tests/test_rolling.py @@ -13,7 +13,6 @@ assert_equal, assert_identical, dask_array_api, - has_dask, requires_bottleneck, requires_dask, requires_dask_ge_2024_11_0, @@ -110,17 +109,17 @@ def test_rolling_properties(self, da) -> None: ): da.rolling(foo=2) - @requires_dask + @requires_bottleneck @pytest.mark.parametrize( "name", ("sum", "mean", "std", "min", "max", "median", "argmin", "argmax") ) @pytest.mark.parametrize("center", (True, False, None)) @pytest.mark.parametrize("min_periods", (1, None)) - @pytest.mark.parametrize("backend", ["numpy", "dask"], indirect=True) def test_rolling_wrapped_bottleneck( self, da, name, center, min_periods, compute_backend ) -> None: - bn = pytest.importorskip("bottleneck", minversion="1.1") + import bottleneck as bn + # Test all bottleneck functions rolling_obj = da.rolling(time=7, min_periods=min_periods) @@ -153,8 +152,10 @@ def test_rolling_wrapped_bottleneck( @pytest.mark.parametrize("center", (True, False, None)) @pytest.mark.parametrize("min_periods", (1, None)) @pytest.mark.parametrize("window", (7, 8)) - @pytest.mark.parametrize("backend", ["dask"], indirect=True) - def test_rolling_wrapped_dask(self, da, name, center, min_periods, window) -> None: + def test_rolling_wrapped_dask( + self, da_numpy, name, center, min_periods, window + ) -> None: + da = da_numpy.chunk() # dask version rolling_obj = da.rolling(time=window, min_periods=min_periods, center=center) actual = getattr(rolling_obj, name)().load() @@ -178,11 +179,10 @@ def test_rolling_wrapped_dask(self, da, name, center, min_periods, window) -> No actual = getattr(rolling_obj, name)().load() assert_allclose(actual, expected) + @requires_dask @pytest.mark.parametrize("center", (True, None)) def test_rolling_wrapped_dask_nochunk(self, center) -> None: # GH:2113 - pytest.importorskip("dask.array") - da_day_clim = xr.DataArray( np.arange(1, 367), coords=[np.arange(1, 367)], dims="dayofyear" ) @@ -438,17 +438,36 @@ def test_rolling_dask_dtype(self, dtype) -> None: @requires_bottleneck @requires_dask_ge_2024_11_0 @pytest.mark.parametrize( - "name", - ("sum", "mean", "std", "var", "min", "max", "median", "argmin", "argmax"), + "name, center, dtype", + [ + pytest.param( + name, + center, + dtype, + marks=pytest.mark.skip( + reason="centered bool bottleneck path fails for numpy-backed arrays" + ) + if center and dtype is bool and name in ("std", "median") + else (), + ) + for name in ( + "sum", + "mean", + "std", + "var", + "min", + "max", + "median", + "argmin", + "argmax", + ) + for center in (False, True) + for dtype in (bool, np.int8, np.int64) + ], ) - @pytest.mark.parametrize("center", [False, True]) - @pytest.mark.parametrize("dtype", [bool, np.int8, np.int64]) def test_rolling_bottleneck_dask_dtype_matches_numpy( self, name, center, dtype ) -> None: - if center and dtype is bool and name in ("std", "median"): - pytest.skip("centered bool bottleneck path fails for numpy-backed arrays") - raw = np.arange(100 * 4).reshape(100, 4) data = raw % 3 == 0 if dtype is bool else raw.astype(dtype) unchunked = DataArray(data, dims=("t", "a")).rolling( @@ -503,10 +522,9 @@ class TestDataArrayRollingExp: "window_type, window", [["span", 5], ["alpha", 0.5], ["com", 0.5], ["halflife", 5]], ) - @pytest.mark.parametrize("backend", ["numpy"], indirect=True) @pytest.mark.parametrize("func", ["mean", "sum", "var", "std"]) - def test_rolling_exp_runs(self, da, dim, window_type, window, func) -> None: - da = da.where(da > 0.2) + def test_rolling_exp_runs(self, da_numpy, dim, window_type, window, func) -> None: + da = da_numpy.where(da_numpy > 0.2) rolling_exp = da.rolling_exp(window_type=window_type, **{dim: window}) result = getattr(rolling_exp, func)() @@ -517,9 +535,8 @@ def test_rolling_exp_runs(self, da, dim, window_type, window, func) -> None: "window_type, window", [["span", 5], ["alpha", 0.5], ["com", 0.5], ["halflife", 5]], ) - @pytest.mark.parametrize("backend", ["numpy"], indirect=True) - def test_rolling_exp_mean_pandas(self, da, dim, window_type, window) -> None: - da = da.isel(a=0).where(lambda x: x > 0.2) + def test_rolling_exp_mean_pandas(self, da_numpy, dim, window_type, window) -> None: + da = da_numpy.isel(a=0).where(lambda x: x > 0.2) result = da.rolling_exp(window_type=window_type, **{dim: window}).mean() assert isinstance(result, DataArray) @@ -534,14 +551,13 @@ def test_rolling_exp_mean_pandas(self, da, dim, window_type, window) -> None: assert_allclose(expected.variable, result.variable) - @pytest.mark.parametrize("backend", ["numpy"], indirect=True) @pytest.mark.parametrize("func", ["mean", "sum"]) - def test_rolling_exp_keep_attrs(self, da, func) -> None: + def test_rolling_exp_keep_attrs(self, da_numpy, func) -> None: attrs = {"attrs": "da"} - da.attrs = attrs + da_numpy.attrs = attrs - # Equivalent of `da.rolling_exp(time=10).mean` - rolling_exp_func = getattr(da.rolling_exp(time=10), func) + # Equivalent of `da_numpy.rolling_exp(time=10).mean` + rolling_exp_func = getattr(da_numpy.rolling_exp(time=10), func) # attrs are kept per default result = rolling_exp_func() @@ -569,7 +585,7 @@ def test_rolling_exp_keep_attrs(self, da, func) -> None: UserWarning, match="Passing ``keep_attrs`` to ``rolling_exp`` has no effect.", ): - da.rolling_exp(time=10, keep_attrs=True) + da_numpy.rolling_exp(time=10, keep_attrs=True) class TestDatasetRolling: @@ -698,37 +714,37 @@ def test_rolling_construct_automatic_rechunk(self): ) assert_identical(one, two) + @requires_bottleneck @pytest.mark.parametrize( "name", ("sum", "mean", "std", "var", "min", "max", "median") ) @pytest.mark.parametrize("center", (True, False, None)) @pytest.mark.parametrize("min_periods", (1, None)) @pytest.mark.parametrize("key", ("z1", "z2")) - @pytest.mark.parametrize("backend", ["numpy"], indirect=True) def test_rolling_wrapped_bottleneck( - self, ds, name, center, min_periods, key, compute_backend + self, ds_numpy, name, center, min_periods, key, compute_backend ) -> None: - bn = pytest.importorskip("bottleneck", minversion="1.1") + import bottleneck as bn # Test all bottleneck functions - rolling_obj = ds.rolling(time=7, min_periods=min_periods) + rolling_obj = ds_numpy.rolling(time=7, min_periods=min_periods) func_name = f"move_{name}" actual = getattr(rolling_obj, name)() if key == "z1": # z1 does not depend on 'Time' axis. Stored as it is. - expected = ds[key] + expected = ds_numpy[key] elif key == "z2": expected = getattr(bn, func_name)( - ds[key].values, window=7, axis=0, min_count=min_periods + ds_numpy[key].values, window=7, axis=0, min_count=min_periods ) else: raise ValueError np.testing.assert_allclose(actual[key].values, expected) # Test center - rolling_obj = ds.rolling(time=7, center=center) + rolling_obj = ds_numpy.rolling(time=7, center=center) actual = getattr(rolling_obj, name)()["time"] - assert_allclose(actual, ds["time"]) + assert_allclose(actual, ds_numpy["time"]) @pytest.mark.parametrize("center", (True, False)) @pytest.mark.parametrize("min_periods", (None, 1, 2, 3)) @@ -833,17 +849,26 @@ def test_rolling_construct_stride(self, center: bool, window: int) -> None: @pytest.mark.parametrize("ds", (1, 2), indirect=True) @pytest.mark.parametrize("center", (True, False)) @pytest.mark.parametrize("min_periods", (None, 1, 2, 3)) - @pytest.mark.parametrize("window", (1, 2, 3, 4)) @pytest.mark.parametrize( - "name", ("sum", "mean", "std", "var", "min", "max", "median") + "name, window", + [ + pytest.param( + name, + window, + marks=pytest.mark.skip( + reason="std with window == 1 is unstable in bottleneck" + ) + if name == "std" and window == 1 + else (), + ) + for name in ("sum", "mean", "std", "var", "min", "max", "median") + for window in (1, 2, 3, 4) + ], ) def test_rolling_reduce(self, ds, center, min_periods, window, name) -> None: if min_periods is not None and window < min_periods: min_periods = window - if name == "std" and window == 1: - pytest.skip("std with window == 1 is unstable in bottleneck") - rolling_obj = ds.rolling(time=window, center=center, min_periods=min_periods) # add nan prefix to numpy methods to get similar behavior as bottleneck @@ -862,9 +887,8 @@ def test_rolling_reduce(self, ds, center, min_periods, window, name) -> None: @pytest.mark.parametrize("center", (True, False)) @pytest.mark.parametrize("min_periods", (None, 1)) @pytest.mark.parametrize("name", ("sum", "max")) - @pytest.mark.parametrize("dask", (True, False)) - def test_ndrolling_reduce(self, ds, center, min_periods, name, dask) -> None: - if dask and has_dask: + def test_ndrolling_reduce(self, ds, center, min_periods, name, use_dask) -> None: + if use_dask: ds = ds.chunk({"x": 4}) rolling_obj = ds.rolling(time=4, x=3, center=center, min_periods=min_periods) @@ -892,15 +916,14 @@ def test_ndrolling_reduce(self, ds, center, min_periods, name, dask) -> None: @pytest.mark.parametrize("center", (True, False, (True, False))) @pytest.mark.parametrize("fill_value", (np.nan, 0.0)) - @pytest.mark.parametrize("dask", (True, False)) - def test_ndrolling_construct(self, center, fill_value, dask) -> None: + def test_ndrolling_construct(self, center, fill_value, use_dask) -> None: da = DataArray( np.arange(5 * 6 * 7).reshape(5, 6, 7).astype(float), dims=["x", "y", "z"], coords={"x": ["a", "b", "c", "d", "e"], "y": np.arange(6)}, ) ds = xr.Dataset({"da": da}) - if dask and has_dask: + if use_dask: ds = ds.chunk({"x": 4}) actual = ds.rolling(x=3, z=2, center=center).construct( @@ -947,28 +970,24 @@ def test_rolling_keepdims_warns(self, func) -> None: @requires_numbagg class TestDatasetRollingExp: - @pytest.mark.parametrize( - "backend", ["numpy", pytest.param("dask", marks=requires_dask)], indirect=True - ) def test_rolling_exp(self, ds) -> None: result = ds.rolling_exp(time=10, window_type="span").mean() assert isinstance(result, Dataset) - @pytest.mark.parametrize("backend", ["numpy"], indirect=True) - def test_rolling_exp_keep_attrs(self, ds) -> None: + def test_rolling_exp_keep_attrs(self, ds_numpy) -> None: attrs_global = {"attrs": "global"} attrs_z1 = {"attr": "z1"} - ds.attrs = attrs_global - ds.z1.attrs = attrs_z1 + ds_numpy.attrs = attrs_global + ds_numpy.z1.attrs = attrs_z1 # attrs are kept per default - result = ds.rolling_exp(time=10).mean() + result = ds_numpy.rolling_exp(time=10).mean() assert result.attrs == attrs_global assert result.z1.attrs == attrs_z1 # discard attrs - result = ds.rolling_exp(time=10).mean(keep_attrs=False) + result = ds_numpy.rolling_exp(time=10).mean(keep_attrs=False) assert result.attrs == {} # TODO: from #8114 — this arguably should be empty, but `apply_ufunc` doesn't do # that at the moment. We should change in `apply_func` rather than @@ -978,19 +997,19 @@ def test_rolling_exp_keep_attrs(self, ds) -> None: # test discard attrs using global option with set_options(keep_attrs=False): - result = ds.rolling_exp(time=10).mean() + result = ds_numpy.rolling_exp(time=10).mean() assert result.attrs == {} # See above # assert result.z1.attrs == {} # keyword takes precedence over global option with set_options(keep_attrs=False): - result = ds.rolling_exp(time=10).mean(keep_attrs=True) + result = ds_numpy.rolling_exp(time=10).mean(keep_attrs=True) assert result.attrs == attrs_global assert result.z1.attrs == attrs_z1 with set_options(keep_attrs=True): - result = ds.rolling_exp(time=10).mean(keep_attrs=False) + result = ds_numpy.rolling_exp(time=10).mean(keep_attrs=False) assert result.attrs == {} # See above # assert result.z1.attrs == {} @@ -999,4 +1018,4 @@ def test_rolling_exp_keep_attrs(self, ds) -> None: UserWarning, match="Passing ``keep_attrs`` to ``rolling_exp`` has no effect.", ): - ds.rolling_exp(time=10, keep_attrs=True) + ds_numpy.rolling_exp(time=10, keep_attrs=True) diff --git a/xarray/tests/test_strategies.py b/xarray/tests/test_strategies.py index b4cf8bf2c16..ded70dd463e 100644 --- a/xarray/tests/test_strategies.py +++ b/xarray/tests/test_strategies.py @@ -273,13 +273,20 @@ def test_mean(self, data, var): Test that given a Variable of at least one dimension, the mean of the Variable is always equal to the mean of the underlying array. """ - with set_options(use_numbagg=False): + # arbitrary floats can overflow to inf, and inf - inf gives nan + with ( + set_options(use_numbagg=False), + np.errstate(over="ignore", invalid="ignore"), + ): # specify arbitrary reduction along at least one dimension reduction_dims = data.draw(unique_subset_of(var.dims, min_size=1)) # create expected result (using nanmean because arrays with Nans will be generated) reduction_axes = tuple(var.get_axis_num(dim) for dim in reduction_dims) - expected = np.nanmean(var.data, axis=reduction_axes) + with warnings.catch_warnings(): + # all-NaN slices are expected to reduce to NaN + warnings.filterwarnings("ignore", "Mean of empty slice", RuntimeWarning) + expected = np.nanmean(var.data, axis=reduction_axes) # assert property is always satisfied result = var.mean(dim=reduction_dims).data diff --git a/xarray/tests/test_ufuncs.py b/xarray/tests/test_ufuncs.py index f4325fa5cea..a5a73480cb5 100644 --- a/xarray/tests/test_ufuncs.py +++ b/xarray/tests/test_ufuncs.py @@ -9,7 +9,13 @@ import xarray as xr import xarray.ufuncs as xu -from xarray.tests import assert_allclose, assert_array_equal, mock, requires_dask +from xarray.tests import ( + assert_allclose, + assert_array_equal, + mock, + requires_dask, + requires_numpy_2, +) from xarray.tests import assert_identical as assert_identical_ @@ -131,8 +137,10 @@ def __array_ufunc__(self, *args, **kwargs): assert np.sin(xarray_obj, out=other) == "other" # type: ignore[call-overload] +@requires_dask def test_xarray_handles_dask(): - da = pytest.importorskip("dask.array") + import dask.array as da + x = xr.DataArray(np.ones((2, 2)), dims=["x", "y"]) y = da.ones((2, 2), chunks=(2, 2)) result = np.add(x, y) @@ -140,8 +148,10 @@ def test_xarray_handles_dask(): assert isinstance(result, xr.DataArray) +@requires_dask def test_dask_defers_to_xarray(): - da = pytest.importorskip("dask.array") + import dask.array as da + x = xr.DataArray(np.ones((2, 2)), dims=["x", "y"]) y = da.ones((2, 2), chunks=(2, 2)) result = np.add(y, x) @@ -208,12 +218,16 @@ def setUp(self): self.xt = xr.DataArray(np.datetime64("2021-01-01", "ns")) @pytest.mark.filterwarnings("ignore::RuntimeWarning") - @pytest.mark.parametrize("name", xu.__all__) + @pytest.mark.parametrize( + "name", + [ + pytest.param(name, marks=() if hasattr(np, name) else requires_numpy_2) + for name in xu.__all__ + ], + ) def test_ufuncs(self, name, request): xu_func = getattr(xu, name) np_func = getattr(np, name, None) - if np_func is None and np.lib.NumpyVersion(np.__version__) < "2.0.0": - pytest.skip(f"Ufunc {name} is not available in numpy {np.__version__}.") if name == "isnat": args = (self.xt,) diff --git a/xarray/tests/test_units.py b/xarray/tests/test_units.py index 42bf3649202..f91ee83873c 100644 --- a/xarray/tests/test_units.py +++ b/xarray/tests/test_units.py @@ -863,8 +863,14 @@ def test_combine_nested(variant, unit, error, dtype): ) ds3 = xr.Dataset( data_vars={ - "a": (("y", "x"), np.full_like(array1, fill_value=np.nan) * data_unit2), - "b": (("y", "x"), np.full_like(array2, fill_value=np.nan) * data_unit2), + "a": ( + ("y", "x"), + np.full_like(array1, fill_value=np.nan, dtype=np.float64) * data_unit2, + ), + "b": ( + ("y", "x"), + np.full_like(array2, fill_value=np.nan, dtype=np.float64) * data_unit2, + ), }, coords={ "x": np.arange(3, 6) * dim_unit2, @@ -1208,8 +1214,14 @@ def test_merge_dataset(variant, unit, error, dtype): ) ds3 = xr.Dataset( data_vars={ - "a": (("y", "x"), np.full_like(array1, np.nan) * data_unit2), - "b": (("y", "x"), np.full_like(array2, np.nan) * data_unit2), + "a": ( + ("y", "x"), + np.full_like(array1, np.nan, dtype=np.float64) * data_unit2, + ), + "b": ( + ("y", "x"), + np.full_like(array2, np.nan, dtype=np.float64) * data_unit2, + ), }, coords={ "x": np.arange(3, 6) * dim_unit2, @@ -1856,7 +1868,6 @@ def test_broadcast_equals(self, unit, dtype): assert expected == actual - @pytest.mark.parametrize("dask", [False, pytest.param(True, marks=[requires_dask])]) @pytest.mark.parametrize( ["variable", "indexers"], ( @@ -1882,8 +1893,8 @@ def test_broadcast_equals(self, unit, dtype): ), ), ) - def test_isel(self, variable, indexers, dask, dtype): - if dask: + def test_isel(self, variable, indexers, use_dask, dtype): + if use_dask: variable = variable.chunk(dict.fromkeys(variable.dims, 2)) quantified = xr.Variable( variable.dims, variable.data.astype(dtype) * unit_registry.s diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 26fba9fcc7d..1481fb5c548 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1949,9 +1949,6 @@ def test_quantile_dask(self, q, axis, dim): np.testing.assert_allclose(actual.values, expected) @pytest.mark.parametrize("method", ["midpoint", "lower"]) - @pytest.mark.parametrize( - "use_dask", [pytest.param(True, marks=requires_dask), False] - ) def test_quantile_method(self, method, use_dask) -> None: v = Variable(["x", "y"], self.d) if use_dask: @@ -2510,10 +2507,11 @@ def test_dask_rolling(self, dim, window, center): assert actual.shape == expected.shape assert_equal(actual, expected) + @pytest.mark.skipif( + not has_dask_array_expr, + reason="only meaningful with an alternate dask chunk manager", + ) def test_legacy_dask_array_rejected_by_dask_array_manager(self): - if not has_dask_array_expr: - pytest.skip("only meaningful with an alternate dask chunk manager") - import dask.array as da x = Variable("x", da.arange(6, chunks=3)) @@ -3127,33 +3125,6 @@ def test_from_pint(self, Var): assert_identical(v.as_numpy(), Var("x", arr)) np.testing.assert_equal(v.to_numpy(), arr) - @requires_sparse - def test_from_sparse(self, Var): - if Var is IndexVariable: - pytest.skip("Can't have 2D IndexVariables") - - import sparse - - arr = np.diagflat([1, 2, 3]) - coords = np.array([[0, 1, 2], [0, 1, 2]]) - sparr = sparse.COO(coords=coords, data=[1, 2, 3], shape=(3, 3)) - v = Variable(["x", "y"], sparr) - - assert_identical(v.as_numpy(), Variable(["x", "y"], arr)) - np.testing.assert_equal(v.to_numpy(), arr) - - @requires_cupy - def test_from_cupy(self, Var): - if Var is IndexVariable: - pytest.skip("cupy in default indexes is not supported at the moment") - import cupy as cp - - arr = np.array([1, 2, 3]) - v = Var("x", cp.array(arr)) - - assert_identical(v.as_numpy(), Var("x", arr)) - np.testing.assert_equal(v.to_numpy(), arr) - @requires_dask @requires_pint @pytest.mark.skip_with_dask_array @@ -3174,6 +3145,33 @@ def test_from_pint_wrapping_dask(self, Var): np.testing.assert_equal(v.to_numpy(), arr) +# Not part of TestNumpyCoercion, as these only apply to Variable, not IndexVariable +@requires_sparse +def test_numpy_coercion_from_sparse() -> None: + # Can't have 2D IndexVariables + import sparse + + arr = np.diagflat([1, 2, 3]) + coords = np.array([[0, 1, 2], [0, 1, 2]]) + sparr = sparse.COO(coords=coords, data=[1, 2, 3], shape=(3, 3)) + v = Variable(["x", "y"], sparr) + + assert_identical(v.as_numpy(), Variable(["x", "y"], arr)) + np.testing.assert_equal(v.to_numpy(), arr) + + +@requires_cupy +def test_numpy_coercion_from_cupy() -> None: + # cupy in default indexes is not supported at the moment + import cupy as cp + + arr = np.array([1, 2, 3]) + v = Variable("x", cp.array(arr)) + + assert_identical(v.as_numpy(), Variable("x", arr)) + np.testing.assert_equal(v.to_numpy(), arr) + + @pytest.mark.parametrize( ("values", "unit"), [