From e045bd3e1dcbeedd31205d8ceb4186e28edc6cc2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 6 Sep 2020 18:23:52 +0200 Subject: [PATCH 01/11] un-xfail the pint assert_allclose and assert_duckarray_equal tests --- xarray/tests/test_testing.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index 0f2ae8b31d4..30ea6aaaee9 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -70,12 +70,7 @@ def test_assert_allclose(obj1, obj2): pytest.param( quantity, id="pint", - marks=[ - pytest.mark.skipif(not has_pint, reason="requires pint"), - pytest.mark.xfail( - reason="inconsistencies in the return value of pint's implementation of eq" - ), - ], + marks=pytest.mark.skipif(not has_pint, reason="requires pint"), ), ), ) @@ -115,12 +110,7 @@ def test_assert_duckarray_equal_failing(duckarray, obj1, obj2): pytest.param( quantity, id="pint", - marks=[ - pytest.mark.skipif(not has_pint, reason="requires pint"), - pytest.mark.xfail( - reason="inconsistencies in the return value of pint's implementation of eq" - ), - ], + marks=pytest.mark.skipif(not has_pint, reason="requires pint"), ), ), ) From d3f3257ec6ed7039aeaec3849ed15b12cccb53c7 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 6 Sep 2020 19:25:43 +0200 Subject: [PATCH 02/11] update the required version of pint --- ci/requirements/py36-min-nep18.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/py36-min-nep18.yml b/ci/requirements/py36-min-nep18.yml index 17aae6932ac..14982c1d5e7 100644 --- a/ci/requirements/py36-min-nep18.yml +++ b/ci/requirements/py36-min-nep18.yml @@ -10,7 +10,7 @@ dependencies: - distributed=2.9 - numpy=1.17 - pandas=0.25 - - pint=0.13 + - pint=0.15 - pip - pytest - pytest-cov From 0fe9c21511345e59edaeee83ec12e42d89d42ed3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 11 Sep 2020 19:47:27 +0200 Subject: [PATCH 03/11] keep the order of the coordinates --- xarray/core/dataset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 825d2044a12..28c2ab13e58 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1123,6 +1123,7 @@ def _copy_listed(self, names: Iterable[Hashable]) -> "Dataset": coord_names = set() indexes: Dict[Hashable, pd.Index] = {} + names = [name for name in self.variables.keys() if name in names] for name in names: try: variables[name] = self._variables[name] @@ -1142,7 +1143,10 @@ def _copy_listed(self, names: Iterable[Hashable]) -> "Dataset": dims = {k: self.dims[k] for k in needed_dims} - for k in self._coord_names: + for k in self._variables: + if k not in self._coord_names: + continue + if set(self.variables[k].dims) <= needed_dims: variables[k] = self._variables[k] coord_names.add(k) From 1de39331744912770a8c59c0db21cd90eebde40d Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 11 Sep 2020 20:10:38 +0200 Subject: [PATCH 04/11] fix the groupby doctest --- xarray/core/dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 28c2ab13e58..fcc0ca26677 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -5733,10 +5733,10 @@ def filter_by_attrs(self, **kwargs): Dimensions: (time: 3, x: 2, y: 2) Coordinates: - reference_time datetime64[ns] 2014-09-05 + lon (x, y) float64 -99.83 -99.32 -99.79 -99.23 lat (x, y) float64 42.25 42.21 42.63 42.59 * time (time) datetime64[ns] 2014-09-06 2014-09-07 2014-09-08 - lon (x, y) float64 -99.83 -99.32 -99.79 -99.23 + reference_time datetime64[ns] 2014-09-05 Dimensions without coordinates: x, y Data variables: precipitation (x, y, time) float64 5.68 9.256 0.7104 ... 7.992 4.615 7.805 @@ -5746,10 +5746,10 @@ def filter_by_attrs(self, **kwargs): Dimensions: (time: 3, x: 2, y: 2) Coordinates: - reference_time datetime64[ns] 2014-09-05 + lon (x, y) float64 -99.83 -99.32 -99.79 -99.23 lat (x, y) float64 42.25 42.21 42.63 42.59 * time (time) datetime64[ns] 2014-09-06 2014-09-07 2014-09-08 - lon (x, y) float64 -99.83 -99.32 -99.79 -99.23 + reference_time datetime64[ns] 2014-09-05 Dimensions without coordinates: x, y Data variables: temperature (x, y, time) float64 29.11 18.2 22.83 ... 18.28 16.15 26.63 From d53db2a7f8a183ea73306ebb026aaeae33a5c244 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 11 Sep 2020 20:10:59 +0200 Subject: [PATCH 05/11] keep the order of the dims in concat --- xarray/core/concat.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xarray/core/concat.py b/xarray/core/concat.py index 54bc686a322..0246328d4c7 100644 --- a/xarray/core/concat.py +++ b/xarray/core/concat.py @@ -349,7 +349,11 @@ def _parse_datasets( all_coord_names.update(ds.coords) data_vars.update(ds.data_vars) - for dim in set(ds.dims) - dims: + common_dims = set(ds.dims) - dims + for dim in ds.dims: + if dim not in common_dims: + continue + if dim not in dim_coords: dim_coords[dim] = ds.coords[dim].variable dims = dims | set(ds.dims) From 98ad4446a1c19b13b8ed76794971244a4383a2b2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 11 Sep 2020 20:13:43 +0200 Subject: [PATCH 06/11] don't compute a set difference if we're filtering anyways --- xarray/core/concat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xarray/core/concat.py b/xarray/core/concat.py index 0246328d4c7..e8431f3a7b3 100644 --- a/xarray/core/concat.py +++ b/xarray/core/concat.py @@ -349,9 +349,8 @@ def _parse_datasets( all_coord_names.update(ds.coords) data_vars.update(ds.data_vars) - common_dims = set(ds.dims) - dims for dim in ds.dims: - if dim not in common_dims: + if dim in dims: continue if dim not in dim_coords: From 7398198de4fa0c98c3cdc333f0f54f916721131f Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 11 Sep 2020 20:36:49 +0200 Subject: [PATCH 07/11] sort names instead of potentially dropping items --- xarray/core/dataset.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index fcc0ca26677..fa0fe68b688 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1123,8 +1123,13 @@ def _copy_listed(self, names: Iterable[Hashable]) -> "Dataset": coord_names = set() indexes: Dict[Hashable, pd.Index] = {} - names = [name for name in self.variables.keys() if name in names] - for name in names: + def key(name): + try: + return list(self._variables.keys()).index(name) + except ValueError: + return len(self._variables) + + for name in sorted(names, key=key): try: variables[name] = self._variables[name] except KeyError: From 42022c109508d646e9f68b82a2607a4c807a0a42 Mon Sep 17 00:00:00 2001 From: keewis Date: Fri, 11 Sep 2020 22:37:24 +0200 Subject: [PATCH 08/11] Apply suggestions from code review Co-authored-by: Deepak Cherian --- xarray/core/concat.py | 1 + xarray/core/dataset.py | 1 + 2 files changed, 2 insertions(+) diff --git a/xarray/core/concat.py b/xarray/core/concat.py index e8431f3a7b3..0955a95fa8b 100644 --- a/xarray/core/concat.py +++ b/xarray/core/concat.py @@ -349,6 +349,7 @@ def _parse_datasets( all_coord_names.update(ds.coords) data_vars.update(ds.data_vars) + # preserves ordering of dimensions for dim in ds.dims: if dim in dims: continue diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index fa0fe68b688..3f871874bec 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1148,6 +1148,7 @@ def key(name): dims = {k: self.dims[k] for k in needed_dims} + # preserves ordering of coordinates for k in self._variables: if k not in self._coord_names: continue From 410e3e55e3a5c1b95ab4a5ace8fa977b61f65a9a Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 12 Sep 2020 00:12:04 +0200 Subject: [PATCH 09/11] sort in DatasetCoordinates.to_dataset instead of in Dataset._copy_listed --- xarray/core/coordinates.py | 10 +++++++++- xarray/core/dataset.py | 8 +------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index a4b8ca478eb..2e63fe3a212 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -215,7 +215,15 @@ def __getitem__(self, key: Hashable) -> "DataArray": def to_dataset(self) -> "Dataset": """Convert these coordinates into a new Dataset""" - return self._data._copy_listed(self._names) + + def key(name): + try: + return list(self._data._variables.keys()).index(name) + except ValueError: + return len(self._data._variables) + + names = sorted(self._names, key=key) + return self._data._copy_listed(names) def _update_coords( self, coords: Dict[Hashable, Variable], indexes: Mapping[Hashable, pd.Index] diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 3f871874bec..ce72d4a5886 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1123,13 +1123,7 @@ def _copy_listed(self, names: Iterable[Hashable]) -> "Dataset": coord_names = set() indexes: Dict[Hashable, pd.Index] = {} - def key(name): - try: - return list(self._variables.keys()).index(name) - except ValueError: - return len(self._variables) - - for name in sorted(names, key=key): + for name in names: try: variables[name] = self._variables[name] except KeyError: From 11e9fd815e1003aef96c3c3aa16889636cda7f5d Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 12 Sep 2020 00:19:32 +0200 Subject: [PATCH 10/11] update whats-new.rst --- doc/whats-new.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 74619529144..6ad6f65e6cc 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,7 +33,7 @@ New Features now accept more than 1 dimension. (:pull:`4219`) By `Keisuke Fujii `_. - ``min_count`` can be supplied to reductions such as ``.sum`` when specifying - multiple dimension to reduce over. (:pull:`4356`) + multiple dimension to reduce over. (:pull:`4356`) By `Maximilian Roos `_. - :py:func:`xarray.cov` and :py:func:`xarray.corr` now handle missing values. (:pull:`4351`) By `Maximilian Roos `_. @@ -77,7 +77,7 @@ Bug fixes and :py:meth:`DataArray.str.wrap` (:issue:`4334`). By `Mathias Hauser `_. - Fixed overflow issue causing incorrect results in computing means of :py:class:`cftime.datetime` arrays (:issue:`4341`). By `Spencer Clark `_. -- Fixed :py:meth:`Dataset.coarsen`, :py:meth:`DataArray.coarsen` dropping attributes on original object (:issue:`4120`, :pull:`4360`). by `Julia Kent `_. +- Fixed :py:meth:`Dataset.coarsen`, :py:meth:`DataArray.coarsen` dropping attributes on original object (:issue:`4120`, :pull:`4360`). By `Julia Kent `_. - fix the signature of the plot methods. (:pull:`4359`) By `Justus Magin `_. - Fix :py:func:`xarray.apply_ufunc` with ``vectorize=True`` and ``exclude_dims`` (:issue:`3890`). By `Mathias Hauser `_. @@ -86,6 +86,8 @@ Bug fixes By `Jens Svensmark `_ - Fix incorrect legend labels for :py:meth:`Dataset.plot.scatter` (:issue:`4126`). By `Peter Hausamann `_. +- Avoid relying on :py:class:`set` objects for the ordering of the coordinates (:pull:`4409`) + By `Justus Magin `_. Documentation ~~~~~~~~~~~~~ @@ -96,6 +98,8 @@ Documentation By `Sander van Rijn `_ - Update the contributing guide to use merges instead of rebasing and state that we squash-merge. (:pull:`4355`) By `Justus Magin `_. +- Make sure the examples from the docstrings actually work (:pull:`4408`). + By `Justus Magin `_. Internal Changes ~~~~~~~~~~~~~~~~ From 371aef6b430f7f75848238bbdbc1344cd1f6ed25 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 12 Sep 2020 11:28:10 +0200 Subject: [PATCH 11/11] filter _variables instead of sorting _coord_names --- xarray/core/coordinates.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index 2e63fe3a212..846e4044a2c 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -216,13 +216,7 @@ def __getitem__(self, key: Hashable) -> "DataArray": def to_dataset(self) -> "Dataset": """Convert these coordinates into a new Dataset""" - def key(name): - try: - return list(self._data._variables.keys()).index(name) - except ValueError: - return len(self._data._variables) - - names = sorted(self._names, key=key) + names = [name for name in self._data._variables if name in self._names] return self._data._copy_listed(names) def _update_coords(