From 327bc54e0eb93609a57c5360a14fcb4e02a35443 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 28 Dec 2018 02:27:58 -0500 Subject: [PATCH 01/32] WIP on ewm using numbagg --- xarray/core/common.py | 6 ++++++ xarray/core/dataarray.py | 3 ++- xarray/core/ewm.py | 26 ++++++++++++++++++++++++++ xarray/tests/test_dataarray.py | 5 +++++ 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 xarray/core/ewm.py diff --git a/xarray/core/common.py b/xarray/core/common.py index 5b090bf0d2f..ef698904cf5 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -590,6 +590,12 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): return self._rolling_cls(self, dim, min_periods=min_periods, center=center) + def ewm(self, dim=None, **dim_kwargs): + + dim = either_dict_or_kwargs(dim, dim_kwargs, 'ewm') + + return self._ewm_cls(self, dim) + def resample(self, indexer=None, skipna=None, closed=None, label=None, base=0, keep_attrs=None, loffset=None, **indexer_kwargs): """Returns a Resample object for performing resampling operations. diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 25a66e529ae..2690ec7c663 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -7,7 +7,7 @@ import pandas as pd from . import ( - computation, dtypes, groupby, indexing, ops, resample, rolling, utils) + computation, dtypes, ewm, groupby, indexing, ops, resample, rolling, utils) from ..plot.plot import _PlotMethods from .accessors import DatetimeAccessor from .alignment import align, reindex_like_indexers @@ -160,6 +160,7 @@ class DataArray(AbstractArray, DataWithCoords): """ _groupby_cls = groupby.DataArrayGroupBy _rolling_cls = rolling.DataArrayRolling + _ewm_cls = ewm.DataArrayEWM _resample_cls = resample.DataArrayResample dt = property(DatetimeAccessor) diff --git a/xarray/core/ewm.py b/xarray/core/ewm.py new file mode 100644 index 00000000000..6a9ed147d56 --- /dev/null +++ b/xarray/core/ewm.py @@ -0,0 +1,26 @@ +from numbagg.moving import ewm_nanmean + +from .computation import apply_ufunc + + +class EWM(object): + _attributes = ['com', 'dim'] + + def __init__(self, obj, spans): + # TODO: add alternatives to com + self.obj = obj + dim, span = next(iter(spans.items())) + self.dim = dim + self.span = span + + +class DataArrayEWM(EWM): + def mean(self): + return apply_ufunc( + ewm_nanmean, + [self.obj], + input_core_dims=(self.dim,), + kwargs=dict(com=self.span), + + + ) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 53f53574031..f1d709fa651 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3846,3 +3846,8 @@ def test_fallback_to_iris_AuxCoord(self, coord_values): expected = Cube(data, aux_coords_and_dims=[ (AuxCoord(coord_values, var_name='space'), 0)]) assert result == expected + + +def test_ewm(da): + result = da.ewm(time=2).mean() + assert isinstance(result, DataArray) \ No newline at end of file From 1a3ae0bcdde9843f602b0bdf33940dc9185eae0b Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 4 Jan 2019 13:18:59 -0500 Subject: [PATCH 02/32] basic functionality, no dims working yet --- xarray/core/ewm.py | 15 ++++++++++----- xarray/tests/test_dataarray.py | 32 +++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/xarray/core/ewm.py b/xarray/core/ewm.py index 6a9ed147d56..c301894a136 100644 --- a/xarray/core/ewm.py +++ b/xarray/core/ewm.py @@ -3,6 +3,12 @@ from .computation import apply_ufunc +def _ewm_nanmean(array, com): + # wrapper b/c of kwarg + # potentially this fuction should be in numbagg? + return ewm_nanmean(array, com) + + class EWM(object): _attributes = ['com', 'dim'] @@ -17,10 +23,9 @@ def __init__(self, obj, spans): class DataArrayEWM(EWM): def mean(self): return apply_ufunc( - ewm_nanmean, - [self.obj], - input_core_dims=(self.dim,), + _ewm_nanmean, + self.obj, + input_core_dims=[[self.dim]], + output_core_dims=[[self.dim]], kwargs=dict(com=self.span), - - ) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index f1d709fa651..2024c52e5db 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3678,14 +3678,14 @@ def test_to_and_from_iris(self): assert coord.var_name == original_coord.name assert_array_equal( coord.points, CFDatetimeCoder().encode(original_coord).values) - assert (actual.coord_dims(coord) == - original.get_axis_num( + assert (actual.coord_dims(coord) + == original.get_axis_num( original.coords[coord.var_name].dims)) - assert (actual.coord('distance2').attributes['foo'] == - original.coords['distance2'].attrs['foo']) - assert (actual.coord('distance').units == - cf_units.Unit(original.coords['distance'].units)) + assert (actual.coord('distance2').attributes['foo'] + == original.coords['distance2'].attrs['foo']) + assert (actual.coord('distance').units + == cf_units.Unit(original.coords['distance'].units)) assert actual.attributes['baz'] == original.attrs['baz'] assert actual.standard_name == original.attrs['standard_name'] @@ -3743,14 +3743,14 @@ def test_to_and_from_iris_dask(self): assert coord.var_name == original_coord.name assert_array_equal( coord.points, CFDatetimeCoder().encode(original_coord).values) - assert (actual.coord_dims(coord) == - original.get_axis_num( + assert (actual.coord_dims(coord) + == original.get_axis_num( original.coords[coord.var_name].dims)) assert (actual.coord('distance2').attributes['foo'] == original.coords[ 'distance2'].attrs['foo']) - assert (actual.coord('distance').units == - cf_units.Unit(original.coords['distance'].units)) + assert (actual.coord('distance').units + == cf_units.Unit(original.coords['distance'].units)) assert actual.attributes['baz'] == original.attrs['baz'] assert actual.standard_name == original.attrs['standard_name'] @@ -3849,5 +3849,15 @@ def test_fallback_to_iris_AuxCoord(self, coord_values): def test_ewm(da): + da = da.isel(a=0) result = da.ewm(time=2).mean() - assert isinstance(result, DataArray) \ No newline at end of file + assert isinstance(result, DataArray) + + pandas_array = da.to_pandas() + assert pandas_array.index.name == 'time' + expected = xr.DataArray(pandas_array.ewm(com=2).mean()) + + # TODO: fix + # expected = expected.transpose('x', 'time') + + assert_equal(expected, result) From d1199052f053c19f70e8aefc9252660714d2dfdb Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 12 Jan 2019 09:55:36 -0500 Subject: [PATCH 03/32] rename to `rolling_exp` --- xarray/core/common.py | 6 +++--- xarray/core/dataarray.py | 11 ++++++----- xarray/core/{ewm.py => rolling_exp.py} | 8 ++++---- xarray/tests/test_dataarray.py | 6 +++--- 4 files changed, 16 insertions(+), 15 deletions(-) rename xarray/core/{ewm.py => rolling_exp.py} (82%) diff --git a/xarray/core/common.py b/xarray/core/common.py index 9f3bbe062da..ff8dbe57f3e 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -590,11 +590,11 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): return self._rolling_cls(self, dim, min_periods=min_periods, center=center) - def ewm(self, dim=None, **dim_kwargs): + def rolling_exp(self, dim=None, **dim_kwargs): - dim = either_dict_or_kwargs(dim, dim_kwargs, 'ewm') + dim = either_dict_or_kwargs(dim, dim_kwargs, 'rolling_exp') - return self._ewm_cls(self, dim) + return self._rolling_exp_cls(self, dim) def coarsen(self, dim=None, boundary='exact', side='left', coord_func='mean', **dim_kwargs): diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index b86e7e1178c..fdcda4a8c44 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -7,17 +7,18 @@ import pandas as pd from . import ( - computation, dtypes, ewm, groupby, indexing, ops, resample, rolling, utils) + computation, dtypes, groupby, indexing, ops, resample, rolling, + rolling_exp, utils) from ..plot.plot import _PlotMethods from .accessors import DatetimeAccessor from .alignment import align, reindex_like_indexers from .common import AbstractArray, DataWithCoords from .coordinates import ( - DataArrayCoordinates, LevelCoordinatesSource, - assert_coordinate_consistent, remap_label_indexers) + DataArrayCoordinates, LevelCoordinatesSource, assert_coordinate_consistent, + remap_label_indexers) from .dataset import Dataset, merge_indexes, split_indexes from .formatting import format_item -from .indexes import default_indexes, Indexes +from .indexes import Indexes, default_indexes from .options import OPTIONS from .pycompat import OrderedDict, basestring, iteritems, range, zip from .utils import ( @@ -161,7 +162,7 @@ class DataArray(AbstractArray, DataWithCoords): """ _groupby_cls = groupby.DataArrayGroupBy _rolling_cls = rolling.DataArrayRolling - _ewm_cls = ewm.DataArrayEWM + _rolling_exp_cls = rolling_exp.DataArrayRollingExp _coarsen_cls = rolling.DataArrayCoarsen _resample_cls = resample.DataArrayResample diff --git a/xarray/core/ewm.py b/xarray/core/rolling_exp.py similarity index 82% rename from xarray/core/ewm.py rename to xarray/core/rolling_exp.py index c301894a136..ad49a4fa59e 100644 --- a/xarray/core/ewm.py +++ b/xarray/core/rolling_exp.py @@ -3,13 +3,13 @@ from .computation import apply_ufunc -def _ewm_nanmean(array, com): +def _rolling_exp_nanmean(array, com): # wrapper b/c of kwarg # potentially this fuction should be in numbagg? return ewm_nanmean(array, com) -class EWM(object): +class RollingExp(object): _attributes = ['com', 'dim'] def __init__(self, obj, spans): @@ -20,10 +20,10 @@ def __init__(self, obj, spans): self.span = span -class DataArrayEWM(EWM): +class DataArrayRollingExp(RollingExp): def mean(self): return apply_ufunc( - _ewm_nanmean, + _rolling_exp_nanmean, self.obj, input_core_dims=[[self.dim]], output_core_dims=[[self.dim]], diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index c565b61fa40..dabaaba5a1f 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3879,14 +3879,14 @@ def test_fallback_to_iris_AuxCoord(self, coord_values): assert result == expected -def test_ewm(da): +def test_rolling_exp(da): da = da.isel(a=0) - result = da.ewm(time=2).mean() + result = da.rolling_exp(time=2).mean() assert isinstance(result, DataArray) pandas_array = da.to_pandas() assert pandas_array.index.name == 'time' - expected = xr.DataArray(pandas_array.ewm(com=2).mean()) + expected = xr.DataArray(pandas_array.rolling_exp(com=2).mean()) # TODO: fix # expected = expected.transpose('x', 'time') From 1ef07f0e3933d1bfafcc592fb6eae9a9a7ef7a1b Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 12 Jan 2019 16:33:43 -0500 Subject: [PATCH 04/32] ensure works on either dimensions --- xarray/core/rolling_exp.py | 3 ++- xarray/tests/test_dataarray.py | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index ad49a4fa59e..6b83f303752 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -22,10 +22,11 @@ def __init__(self, obj, spans): class DataArrayRollingExp(RollingExp): def mean(self): - return apply_ufunc( + da = apply_ufunc( _rolling_exp_nanmean, self.obj, input_core_dims=[[self.dim]], output_core_dims=[[self.dim]], kwargs=dict(com=self.span), ) + return da.transpose(*self.obj.dims) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index dabaaba5a1f..5bd6f5fc79f 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3879,16 +3879,19 @@ def test_fallback_to_iris_AuxCoord(self, coord_values): assert result == expected -def test_rolling_exp(da): +@pytest.mark.parametrize('dim', ['time', 'x']) +def test_rolling_exp(da, dim): da = da.isel(a=0) - result = da.rolling_exp(time=2).mean() + result = da.rolling_exp(**{dim: 2}).mean() assert isinstance(result, DataArray) pandas_array = da.to_pandas() assert pandas_array.index.name == 'time' - expected = xr.DataArray(pandas_array.rolling_exp(com=2).mean()) - - # TODO: fix - # expected = expected.transpose('x', 'time') - - assert_equal(expected, result) + if dim == 'x': + pandas_array = pandas_array.T + expected = ( + xr.DataArray(pandas_array.ewm(com=2).mean()) + .transpose(*da.dims) + ) + + assert_equal(expected.variable, result.variable) From b30a096411fdcb2c2b26a2a3d74d4c3ac89b0864 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sun, 13 Jan 2019 06:28:44 -0500 Subject: [PATCH 05/32] window_type working --- xarray/core/common.py | 4 ++-- xarray/core/rolling_exp.py | 28 ++++++++++++++++------------ xarray/tests/__init__.py | 3 ++- xarray/tests/test_dataarray.py | 18 +++++++++++++----- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index ff8dbe57f3e..8d1f2cca975 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -590,11 +590,11 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): return self._rolling_cls(self, dim, min_periods=min_periods, center=center) - def rolling_exp(self, dim=None, **dim_kwargs): + def rolling_exp(self, dim=None, window_type='span', **dim_kwargs): dim = either_dict_or_kwargs(dim, dim_kwargs, 'rolling_exp') - return self._rolling_exp_cls(self, dim) + return self._rolling_exp_cls(self, dim, window_type) def coarsen(self, dim=None, boundary='exact', side='left', coord_func='mean', **dim_kwargs): diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 6b83f303752..b8032823b9d 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -1,32 +1,36 @@ -from numbagg.moving import ewm_nanmean + +from pandas.core.window import _get_center_of_mass from .computation import apply_ufunc -def _rolling_exp_nanmean(array, com): - # wrapper b/c of kwarg - # potentially this fuction should be in numbagg? - return ewm_nanmean(array, com) +def _get_alpha(com=None, span=None, halflife=None, alpha=None): + # pandas defines in terms of comass, so use its function + # but then convert to alpha + + comass = _get_center_of_mass(com, span, halflife, alpha) + return 1 / (1 + comass) class RollingExp(object): - _attributes = ['com', 'dim'] + _attributes = ['alpha', 'dim'] - def __init__(self, obj, spans): - # TODO: add alternatives to com + def __init__(self, obj, windows, window_type='span'): self.obj = obj - dim, span = next(iter(spans.items())) + dim, window = next(iter(windows.items())) self.dim = dim - self.span = span + self.alpha = _get_alpha(**{window_type: window}) class DataArrayRollingExp(RollingExp): def mean(self): + from numbagg.moving import rolling_exp_nanmean + da = apply_ufunc( - _rolling_exp_nanmean, + rolling_exp_nanmean, self.obj, input_core_dims=[[self.dim]], output_core_dims=[[self.dim]], - kwargs=dict(com=self.span), + kwargs=dict(window=self.alpha), ) return da.transpose(*self.obj.dims) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 58f76596822..bb3396523ff 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -6,7 +6,7 @@ from distutils import version import re import importlib -from unittest import mock +from unittest import mock # noqa import numpy as np from numpy.testing import assert_array_equal # noqa: F401 @@ -76,6 +76,7 @@ def LooseVersion(vstring): has_np113, requires_np113 = _importorskip('numpy', minversion='1.13.0') has_iris, requires_iris = _importorskip('iris') has_cfgrib, requires_cfgrib = _importorskip('cfgrib') +has_numbagg, requires_numbagg = _importorskip('numbagg') # some special cases has_scipy_or_netCDF4 = has_scipy or has_netCDF4 diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 5bd6f5fc79f..6f33e5d5caa 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -21,7 +21,7 @@ LooseVersion, ReturnItem, assert_allclose, assert_array_equal, assert_equal, assert_identical, raises_regex, requires_bottleneck, requires_cftime, requires_dask, requires_iris, requires_np113, - requires_scipy, source_ndarray) + requires_scipy, source_ndarray, requires_numbagg) class TestDataArray(object): @@ -3879,10 +3879,18 @@ def test_fallback_to_iris_AuxCoord(self, coord_values): assert result == expected +@requires_numbagg @pytest.mark.parametrize('dim', ['time', 'x']) -def test_rolling_exp(da, dim): +@pytest.mark.parametrize('window_type, window', [ + ['span', 5], + ['alpha', 0.5], + ['com', 0.5], + ['halflife', 5], +]) +def test_rolling_exp(da, dim, window_type, window): da = da.isel(a=0) - result = da.rolling_exp(**{dim: 2}).mean() + # da = da.where(da > 0.1) + result = da.rolling_exp(window_type=window_type, **{dim: window}).mean() assert isinstance(result, DataArray) pandas_array = da.to_pandas() @@ -3890,8 +3898,8 @@ def test_rolling_exp(da, dim): if dim == 'x': pandas_array = pandas_array.T expected = ( - xr.DataArray(pandas_array.ewm(com=2).mean()) + xr.DataArray(pandas_array.ewm(**{window_type: window}).mean()) .transpose(*da.dims) ) - assert_equal(expected.variable, result.variable) + assert_allclose(expected.variable, result.variable) From 89541b537481be83106e444e2ee23dba2f753ad7 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sun, 13 Jan 2019 10:06:05 -0500 Subject: [PATCH 06/32] add numbagg to travis install --- ci/requirements-py37.yml | 3 +++ xarray/tests/test_dataarray.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 1a98e6b285c..577cff649b3 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -30,3 +30,6 @@ dependencies: - pip: - cfgrib>=0.9.2 - mypy==0.650 + # FIXME: change to a release version before merging + # - git+https://github.com/shoyer/numbagg.git + - git+https://github.com/max-sixty/numbagg.git diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 6f33e5d5caa..6a29820567d 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -21,7 +21,7 @@ LooseVersion, ReturnItem, assert_allclose, assert_array_equal, assert_equal, assert_identical, raises_regex, requires_bottleneck, requires_cftime, requires_dask, requires_iris, requires_np113, - requires_scipy, source_ndarray, requires_numbagg) + requires_numbagg, requires_scipy, source_ndarray) class TestDataArray(object): @@ -3889,7 +3889,8 @@ def test_fallback_to_iris_AuxCoord(self, coord_values): ]) def test_rolling_exp(da, dim, window_type, window): da = da.isel(a=0) - # da = da.where(da > 0.1) + da = da.where(da > 0.2) + result = da.rolling_exp(window_type=window_type, **{dim: window}).mean() assert isinstance(result, DataArray) From 9fe91d2a4623e48153f836a2af5a2b501670a2aa Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sun, 13 Jan 2019 10:11:06 -0500 Subject: [PATCH 07/32] naming --- xarray/core/rolling_exp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index b8032823b9d..3aaaa9cad80 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -5,11 +5,11 @@ def _get_alpha(com=None, span=None, halflife=None, alpha=None): - # pandas defines in terms of comass, so use its function - # but then convert to alpha + # pandas defines in terms of com (converting to alpha in the algo) + # so use its function to get a com and then convert to alpha - comass = _get_center_of_mass(com, span, halflife, alpha) - return 1 / (1 + comass) + com = _get_center_of_mass(com, span, halflife, alpha) + return 1 / (1 + com) class RollingExp(object): From 072b7f8cb0a370d2801267c5f58d7f153574d723 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Mon, 14 Jan 2019 17:04:03 -0500 Subject: [PATCH 08/32] formatting --- xarray/tests/test_dataset.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index e55caf1bf13..91b228dd66d 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function +import pickle import sys import warnings from copy import copy, deepcopy from io import StringIO -import pickle from textwrap import dedent import numpy as np @@ -24,8 +24,8 @@ from . import ( InaccessibleArray, UnexpectedDataAccess, assert_allclose, assert_array_equal, assert_equal, assert_identical, has_cftime, has_dask, - raises_regex, requires_bottleneck, requires_dask, requires_scipy, - source_ndarray) + raises_regex, requires_bottleneck, requires_dask, requires_numbagg, + requires_scipy, source_ndarray) try: import dask.array as da @@ -4513,6 +4513,13 @@ def test_rolling_wrapped_bottleneck(ds, name, center, min_periods, key): assert_equal(actual, ds['time']) +@requires_numbagg +def test_rolling_exp(ds): + + result = ds.rolling_exp(time=10, window_type='span').mean() + assert isinstance(result, Dataset) + + @pytest.mark.parametrize('center', (True, False)) @pytest.mark.parametrize('min_periods', (None, 1, 2, 3)) @pytest.mark.parametrize('window', (1, 2, 3, 4)) From bf7ac74587a6e26fb57d25430cac6598d5aea1be Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 15 Jan 2019 07:59:54 -0500 Subject: [PATCH 09/32] @shoyer's function to abstract the type of self.obj --- xarray/core/common.py | 3 +++ xarray/core/dataarray.py | 4 +--- xarray/core/rolling_exp.py | 26 ++++++++++++-------------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index 8d1f2cca975..7126d3ea1c6 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -9,6 +9,7 @@ from .arithmetic import SupportsArithmetic from .options import _get_keep_attrs from .pycompat import OrderedDict, basestring, dask_array_type, suppress +from .rolling_exp import RollingExp from .utils import Frozen, ReprObject, SortedKeysDict, either_dict_or_kwargs # Used as a sentinel value to indicate a all dimensions @@ -243,6 +244,8 @@ def get_squeeze_dims(xarray_obj, dim, axis=None): class DataWithCoords(SupportsArithmetic, AttrAccessMixin): """Shared base class for Dataset and DataArray.""" + _rolling_exp_cls = RollingExp + def squeeze(self, dim=None, drop=False, axis=None): """Return a new object with squeezed data. diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index fdcda4a8c44..f11a3ade77d 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -7,8 +7,7 @@ import pandas as pd from . import ( - computation, dtypes, groupby, indexing, ops, resample, rolling, - rolling_exp, utils) + computation, dtypes, groupby, indexing, ops, resample, rolling, utils) from ..plot.plot import _PlotMethods from .accessors import DatetimeAccessor from .alignment import align, reindex_like_indexers @@ -162,7 +161,6 @@ class DataArray(AbstractArray, DataWithCoords): """ _groupby_cls = groupby.DataArrayGroupBy _rolling_cls = rolling.DataArrayRolling - _rolling_exp_cls = rolling_exp.DataArrayRollingExp _coarsen_cls = rolling.DataArrayCoarsen _resample_cls = resample.DataArrayResample diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 3aaaa9cad80..d6a464b3450 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -1,8 +1,7 @@ +import numpy as np from pandas.core.window import _get_center_of_mass -from .computation import apply_ufunc - def _get_alpha(com=None, span=None, halflife=None, alpha=None): # pandas defines in terms of com (converting to alpha in the algo) @@ -12,6 +11,15 @@ def _get_alpha(com=None, span=None, halflife=None, alpha=None): return 1 / (1 + com) +def rolling_exp_nanmean(array, *, axis, window): + import numbagg + if axis == (): + return array.astype(np.float) + else: + return numbagg.moving.rolling_exp_nanmean( + array, axis=axis, window=window) + + class RollingExp(object): _attributes = ['alpha', 'dim'] @@ -21,16 +29,6 @@ def __init__(self, obj, windows, window_type='span'): self.dim = dim self.alpha = _get_alpha(**{window_type: window}) - -class DataArrayRollingExp(RollingExp): def mean(self): - from numbagg.moving import rolling_exp_nanmean - - da = apply_ufunc( - rolling_exp_nanmean, - self.obj, - input_core_dims=[[self.dim]], - output_core_dims=[[self.dim]], - kwargs=dict(window=self.alpha), - ) - return da.transpose(*self.obj.dims) + return self.obj.reduce( + rolling_exp_nanmean, dim=self.dim, window=self.alpha) From 061ef870585163777e854bca13a6e3abb6d024f9 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 15 Jan 2019 10:29:56 -0500 Subject: [PATCH 10/32] initial docstring --- xarray/core/rolling.py | 6 ++++-- xarray/core/rolling_exp.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 57463ef5987..8affcba911f 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -8,8 +8,8 @@ from . import dtypes, duck_array_ops, utils from .dask_array_ops import dask_rolling_wrapper from .ops import ( - bn, has_bottleneck, inject_coarsen_methods, - inject_bottleneck_rolling_methods, inject_datasetrolling_methods) + bn, has_bottleneck, inject_bottleneck_rolling_methods, + inject_coarsen_methods, inject_datasetrolling_methods) from .pycompat import OrderedDict, dask_array_type, zip @@ -501,6 +501,7 @@ def _reduce_method(cls, func): Return a wrapped function for injecting numpy methods. see ops.inject_coarsen_methods """ + def wrapped_func(self, **kwargs): from .dataarray import DataArray @@ -529,6 +530,7 @@ def _reduce_method(cls, func): Return a wrapped function for injecting numpy methods. see ops.inject_coarsen_methods """ + def wrapped_func(self, **kwargs): from .dataset import Dataset diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index d6a464b3450..0508653f456 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -21,6 +21,29 @@ def rolling_exp_nanmean(array, *, axis, window): class RollingExp(object): + """ + Exponentially-weighted moving window object. + + Parameters + ---------- + obj : Dataset or DataArray + Object to window. + windows : A mapping from a dimension name to window value + dim : str + Name of the dimension to create the rolling exponential window + along (e.g., `time`). + window : int + Size of the moving window. The type of this is specified in + `window_type` + window_type : str, one of ['span', 'com', 'halflife', 'alpha'], default 'span' + The format of the previously supplied window. Each is a simple + numerical transformation of the others. Described in detail: + https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.ewm.html + + Returns + ------- + RollingExp : type of input argument + """ # noqa _attributes = ['alpha', 'dim'] def __init__(self, obj, windows, window_type='span'): @@ -30,5 +53,6 @@ def __init__(self, obj, windows, window_type='span'): self.alpha = _get_alpha(**{window_type: window}) def mean(self): + """Exponentially weighted average""" return self.obj.reduce( rolling_exp_nanmean, dim=self.dim, window=self.alpha) From fe3c9e138564ef71ded917f3c0e4174c923ed12c Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 15 Jan 2019 10:38:20 -0500 Subject: [PATCH 11/32] add docstrings to docs --- doc/api.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/api.rst b/doc/api.rst index e1f70cfbdea..24fbc7f8047 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -147,6 +147,7 @@ Computation Dataset.groupby Dataset.groupby_bins Dataset.rolling + Dataset.rolling_exp Dataset.coarsen Dataset.resample Dataset.diff @@ -313,6 +314,7 @@ Computation DataArray.groupby DataArray.groupby_bins DataArray.rolling + DataArray.rolling_exp DataArray.coarsen DataArray.dt DataArray.resample @@ -529,6 +531,7 @@ Rolling objects core.rolling.DatasetRolling core.rolling.DatasetRolling.construct core.rolling.DatasetRolling.reduce + core.rolling.DatasetRollingExp Resample objects ================ From e45629e8f7c7f68773e147d4a6561437f515a9c3 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 15 Jan 2019 10:44:52 -0500 Subject: [PATCH 12/32] example --- xarray/core/rolling_exp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 0508653f456..9803504c57c 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -53,6 +53,17 @@ def __init__(self, obj, windows, window_type='span'): self.alpha = _get_alpha(**{window_type: window}) def mean(self): - """Exponentially weighted average""" + """ + Exponentially weighted moving average + + Examples + -------- + >>> da = xr.DataArray([1,1,2,2,2], dims='x') + >>> da.rolling_exp(x=2, window_type='span').mean() + + array([1. , 1. , 1.692308, 1.9 , 1.966942]) + Dimensions without coordinates: x + """ + return self.obj.reduce( rolling_exp_nanmean, dim=self.dim, window=self.alpha) From a9809f41f9a4b920169130d255422d10d4debae6 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 15 Jan 2019 16:19:03 -0500 Subject: [PATCH 13/32] correct location for docs --- doc/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api.rst b/doc/api.rst index 24fbc7f8047..0209e7d74a1 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -531,7 +531,7 @@ Rolling objects core.rolling.DatasetRolling core.rolling.DatasetRolling.construct core.rolling.DatasetRolling.reduce - core.rolling.DatasetRollingExp + core.rolling_exp.DatasetRollingExp Resample objects ================ From c6b939bb796012ac1f1a3814e93a2924532ae8fe Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 15 Jan 2019 16:27:17 -0500 Subject: [PATCH 14/32] add numbagg to print_versions --- xarray/util/print_versions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/util/print_versions.py b/xarray/util/print_versions.py index 5459e67e603..bc5207f825a 100755 --- a/xarray/util/print_versions.py +++ b/xarray/util/print_versions.py @@ -109,6 +109,7 @@ def show_versions(as_json=False): ("matplotlib", lambda mod: mod.__version__), ("cartopy", lambda mod: mod.__version__), ("seaborn", lambda mod: mod.__version__), + ("numbagg", lambda mod: mod.__version__), # xarray setup/test ("setuptools", lambda mod: mod.__version__), ("pip", lambda mod: mod.__version__), From eb38403db78c7b9174da1fe557b83467488213a6 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 16 Jan 2019 16:47:41 -0500 Subject: [PATCH 15/32] whatsnew --- doc/whats-new.rst | 6 ++++++ xarray/core/rolling_exp.py | 1 + 2 files changed, 7 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b50df2af10e..2438b79f2d0 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,12 @@ Enhancements - Upsampling an array via interpolation with resample is now dask-compatible, as long as the array is not chunked along the resampling dimension. By `Spencer Clark `_. +- :py:meth:`~xarray.DataArray.rolling_exp` and + :py:meth:`~xarray.Dataset.rolling_exp` added, similar to pandas' + ``pd.DataFrame.ewm`` method. Calling ``.mean`` on the resulting object + will return an exponentially weighted moving average. + By `Maximilian Roos `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 9803504c57c..d769465324f 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -23,6 +23,7 @@ def rolling_exp_nanmean(array, *, axis, window): class RollingExp(object): """ Exponentially-weighted moving window object. + Similar to EWM in pandas Parameters ---------- From 86b411252447ba189e305aae0e2acc03d3f06dd7 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 16 Jan 2019 16:49:05 -0500 Subject: [PATCH 16/32] updating my GH username --- doc/whats-new.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 2438b79f2d0..1b33246b3dd 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -525,7 +525,7 @@ Enhancements arguments in ``data_vars`` to indexes set explicitly in ``coords``, where previously an error would be raised. (:issue:`674`) - By `Maximilian Roos `_. + By `Maximilian Roos `_. - :py:meth:`~DataArray.sel`, :py:meth:`~DataArray.isel` & :py:meth:`~DataArray.reindex`, (and their :py:class:`Dataset` counterparts) now support supplying a ``dict`` @@ -533,12 +533,12 @@ Enhancements of supplying `kwargs`. This allows for more robust behavior of dimension names which conflict with other keyword names, or are not strings. - By `Maximilian Roos `_. + By `Maximilian Roos `_. - :py:meth:`~DataArray.rename` now supports supplying ``**kwargs``, as an alternative to the existing approach of supplying a ``dict`` as the first argument. - By `Maximilian Roos `_. + By `Maximilian Roos `_. - :py:meth:`~DataArray.cumsum` and :py:meth:`~DataArray.cumprod` now support aggregation over multiple dimensions at the same time. This is the default @@ -703,7 +703,7 @@ Enhancements which test each value in the array for whether it is contained in the supplied list, returning a bool array. See :ref:`selecting values with isin` for full details. Similar to the ``np.isin`` function. - By `Maximilian Roos `_. + By `Maximilian Roos `_. - Some speed improvement to construct :py:class:`~xarray.DataArrayRolling` object (:issue:`1993`) By `Keisuke Fujii `_. @@ -1898,7 +1898,7 @@ Enhancements ~~~~~~~~~~~~ - New documentation on :ref:`panel transition`. By - `Maximilian Roos `_. + `Maximilian Roos `_. - New ``Dataset`` and ``DataArray`` methods :py:meth:`~xarray.Dataset.to_dict` and :py:meth:`~xarray.Dataset.from_dict` to allow easy conversion between dictionaries and xarray objects (:issue:`432`). See @@ -1919,9 +1919,9 @@ Bug fixes (:issue:`953`). By `Stephan Hoyer `_. - ``Dataset.__dir__()`` (i.e. the method python calls to get autocomplete options) failed if one of the dataset's keys was not a string (:issue:`852`). - By `Maximilian Roos `_. + By `Maximilian Roos `_. - ``Dataset`` constructor can now take arbitrary objects as values - (:issue:`647`). By `Maximilian Roos `_. + (:issue:`647`). By `Maximilian Roos `_. - Clarified ``copy`` argument for :py:meth:`~xarray.DataArray.reindex` and :py:func:`~xarray.align`, which now consistently always return new xarray objects (:issue:`927`). From 42ccd0c29a548f92b2c22525525486a559085e3b Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 30 Jan 2019 10:50:15 -0500 Subject: [PATCH 17/32] pin to numbagg release --- ci/requirements-py37.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 577cff649b3..2d343e065a1 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -30,6 +30,4 @@ dependencies: - pip: - cfgrib>=0.9.2 - mypy==0.650 - # FIXME: change to a release version before merging - # - git+https://github.com/shoyer/numbagg.git - - git+https://github.com/max-sixty/numbagg.git + - numbagg From 61da1d729b34890789ee149d84e1cb925d04217c Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 30 Jan 2019 11:02:56 -0500 Subject: [PATCH 18/32] rename inner func to move_exp_nanmean --- xarray/core/rolling_exp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index d769465324f..1bd9f6f0c07 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -11,12 +11,12 @@ def _get_alpha(com=None, span=None, halflife=None, alpha=None): return 1 / (1 + com) -def rolling_exp_nanmean(array, *, axis, window): +def move_exp_nanmean(array, *, axis, window): import numbagg if axis == (): return array.astype(np.float) else: - return numbagg.moving.rolling_exp_nanmean( + return numbagg.moving.move_exp_nanmean( array, axis=axis, window=window) @@ -67,4 +67,4 @@ def mean(self): """ return self.obj.reduce( - rolling_exp_nanmean, dim=self.dim, window=self.alpha) + move_exp_nanmean, dim=self.dim, window=self.alpha) From 55ad124e2f429b06ea0e58f4d8249869e5135687 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 30 Jan 2019 11:48:02 -0500 Subject: [PATCH 19/32] merge --- xarray/core/common.py | 2 +- xarray/core/rolling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index c8ce2981127..c270c3faa7f 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -8,7 +8,7 @@ from . import dtypes, duck_array_ops, formatting, ops from .arithmetic import SupportsArithmetic from .options import _get_keep_attrs -from .pycompat import OrderedDict, basestring, dask_array_type, suppress +from .pycompat import dask_array_type from .rolling_exp import RollingExp from .utils import Frozen, ReprObject, SortedKeysDict, either_dict_or_kwargs diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 845bc5dbcbf..7fe7284dc27 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -9,7 +9,7 @@ from .ops import ( bn, has_bottleneck, inject_bottleneck_rolling_methods, inject_coarsen_methods, inject_datasetrolling_methods) -from .pycompat import OrderedDict, dask_array_type, zip +from .pycompat import dask_array_type class Rolling(object): From b6228d3cb1b18eac34e0d059dad30ed769f1c5ce Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 30 Jan 2019 13:58:27 -0500 Subject: [PATCH 20/32] typo --- xarray/tests/test_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 1ac380a66ee..286fc3c26af 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import pickle import sys import warnings from collections import OrderedDict From e85616281cda495aa312c010ec9b56f911fec759 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Mon, 10 Jun 2019 17:21:58 -0400 Subject: [PATCH 21/32] comments from PR --- xarray/core/rolling_exp.py | 43 +++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 1bd9f6f0c07..2cf8aa6b3aa 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -1,6 +1,6 @@ - import numpy as np -from pandas.core.window import _get_center_of_mass + +from .pycompat import dask_array_type def _get_alpha(com=None, span=None, halflife=None, alpha=None): @@ -12,14 +12,51 @@ def _get_alpha(com=None, span=None, halflife=None, alpha=None): def move_exp_nanmean(array, *, axis, window): + if isinstance(array, dask_array_type): + raise TypeError("rolling_exp is not currently support for dask arrays") import numbagg if axis == (): - return array.astype(np.float) + return array.astype(np.float64) else: return numbagg.moving.move_exp_nanmean( array, axis=axis, window=window) +def _get_center_of_mass(comass, span, halflife, alpha): + """ + Vendored from pandas.core.window._get_center_of_mass + + See licenses/PANDAS_LICENSE for the function's license + """ + from pandas.core import common as com + valid_count = com.count_not_none(comass, span, halflife, alpha) + if valid_count > 1: + raise ValueError("comass, span, halflife, and alpha " + "are mutually exclusive") + + # Convert to center of mass; domain checks ensure 0 < alpha <= 1 + if comass is not None: + if comass < 0: + raise ValueError("comass must satisfy: comass >= 0") + elif span is not None: + if span < 1: + raise ValueError("span must satisfy: span >= 1") + comass = (span - 1) / 2. + elif halflife is not None: + if halflife <= 0: + raise ValueError("halflife must satisfy: halflife > 0") + decay = 1 - np.exp(np.log(0.5) / halflife) + comass = 1 / decay - 1 + elif alpha is not None: + if alpha <= 0 or alpha > 1: + raise ValueError("alpha must satisfy: 0 < alpha <= 1") + comass = (1.0 - alpha) / alpha + else: + raise ValueError("Must pass one of comass, span, halflife, or alpha") + + return float(comass) + + class RollingExp(object): """ Exponentially-weighted moving window object. From 102a47320d61c1f6898ca4a586babcecf2caf653 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Mon, 10 Jun 2019 17:32:17 -0400 Subject: [PATCH 22/32] window -> alpha in numbagg --- xarray/core/rolling_exp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 2cf8aa6b3aa..4a78564e86d 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -11,7 +11,7 @@ def _get_alpha(com=None, span=None, halflife=None, alpha=None): return 1 / (1 + com) -def move_exp_nanmean(array, *, axis, window): +def move_exp_nanmean(array, *, axis, alpha): if isinstance(array, dask_array_type): raise TypeError("rolling_exp is not currently support for dask arrays") import numbagg @@ -19,7 +19,7 @@ def move_exp_nanmean(array, *, axis, window): return array.astype(np.float64) else: return numbagg.moving.move_exp_nanmean( - array, axis=axis, window=window) + array, axis=axis, alpha=alpha) def _get_center_of_mass(comass, span, halflife, alpha): @@ -104,4 +104,4 @@ def mean(self): """ return self.obj.reduce( - move_exp_nanmean, dim=self.dim, window=self.alpha) + move_exp_nanmean, dim=self.dim, alpha=self.alpha) From aac0ce9f9220c875dbc12193bea719f089d9aa18 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Mon, 10 Jun 2019 17:41:13 -0400 Subject: [PATCH 23/32] add docs --- doc/computation.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/computation.rst b/doc/computation.rst index 3100925a7d3..d5d98f8dd3d 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -190,6 +190,22 @@ We can also manually iterate through ``Rolling`` objects: for label, arr_window in r: # arr_window is a view of x +.. _comput.rolling_exp: + +While ``rolling`` provides a simple moving average, ``DataArray``s also support +an exponential moving average with :py:meth:`~xarray.DataArray.rolling_exp`. +This is similiar to pandas' ``ewm`` method. numbagg_ is required. + +.. _numbagg: https://github.com/shoyer/numbagg + +.. code:: python + + arr.rolling_exp(y=3).mean() + +The ``rolling_exp`` method takes a ``window_type`` kwarg, which can be ``'alpha'``, +``'com'`` (for ``center-of-mass``), ``'span'``, and ``'halflife'``. The default is +``span``. + Finally, the rolling object has a ``construct`` method which returns a view of the original ``DataArray`` with the windowed dimension in the last position. From b6eebb448d4ef62c161748cb14deaee860e04732 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 11 Jun 2019 00:46:10 -0400 Subject: [PATCH 24/32] doc fix --- doc/computation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/computation.rst b/doc/computation.rst index d5d98f8dd3d..b06d7959504 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -192,7 +192,7 @@ We can also manually iterate through ``Rolling`` objects: .. _comput.rolling_exp: -While ``rolling`` provides a simple moving average, ``DataArray``s also support +While ``rolling`` provides a simple moving average, ``DataArray`` also supports an exponential moving average with :py:meth:`~xarray.DataArray.rolling_exp`. This is similiar to pandas' ``ewm`` method. numbagg_ is required. From 7e5f9d8c3e5d8d5c981dc35f26dbae468e6d32bb Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 11 Jun 2019 00:48:40 -0400 Subject: [PATCH 25/32] whatsnew update --- doc/whats-new.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 718b75ae5d2..4b56f1292dd 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -30,6 +30,11 @@ Enhancements - Add ``fill_value`` argument for reindex, align, and merge operations to enable custom fill values. (:issue:`2876`) By `Zach Griffith `_. +- :py:meth:`~xarray.DataArray.rolling_exp` and + :py:meth:`~xarray.Dataset.rolling_exp` added, similar to pandas' + ``pd.DataFrame.ewm`` method. Calling ``.mean`` on the resulting object + will return an exponentially weighted moving average. + By `Maximilian Roos `_. - Character arrays' character dimension name decoding and encoding handled by ``var.encoding['char_dim_name']`` (:issue:`2895`) By `James McCreight `_. @@ -173,11 +178,6 @@ Other enhancements - Upsampling an array via interpolation with resample is now dask-compatible, as long as the array is not chunked along the resampling dimension. By `Spencer Clark `_. -- :py:meth:`~xarray.DataArray.rolling_exp` and - :py:meth:`~xarray.Dataset.rolling_exp` added, similar to pandas' - ``pd.DataFrame.ewm`` method. Calling ``.mean`` on the resulting object - will return an exponentially weighted moving average. - By `Maximilian Roos `_. - :py:func:`xarray.testing.assert_equal` and :py:func:`xarray.testing.assert_identical` now provide a more detailed From f64a24f667e472eda02825c649c249817ea85972 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 11 Jun 2019 00:49:57 -0400 Subject: [PATCH 26/32] revert formatting changes to unchanged file --- xarray/core/rolling.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index e8c672d459c..a884963bf06 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -7,8 +7,8 @@ from . import dtypes, duck_array_ops, utils from .dask_array_ops import dask_rolling_wrapper from .ops import ( - bn, has_bottleneck, inject_bottleneck_rolling_methods, - inject_coarsen_methods, inject_datasetrolling_methods) + bn, has_bottleneck, inject_coarsen_methods, + inject_bottleneck_rolling_methods, inject_datasetrolling_methods) from .pycompat import dask_array_type @@ -523,7 +523,6 @@ def _reduce_method(cls, func): Return a wrapped function for injecting numpy methods. see ops.inject_coarsen_methods """ - def wrapped_func(self, **kwargs): from .dataarray import DataArray @@ -552,7 +551,6 @@ def _reduce_method(cls, func): Return a wrapped function for injecting numpy methods. see ops.inject_coarsen_methods """ - def wrapped_func(self, **kwargs): from .dataset import Dataset From ac299c25b1746295ebc9146e9abdf97ccf2d4a9f Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 21 Jun 2019 00:52:27 -0400 Subject: [PATCH 27/32] update docstrings, adjust kwarg names --- xarray/core/common.py | 36 +++++++++++++++++++++++++++++++++--- xarray/core/rolling_exp.py | 4 ++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index 1421d33e633..c589de403e0 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -87,6 +87,7 @@ def wrapped_func(self, dim=None, **kwargs): # type: ignore class AbstractArray(ImplementsArrayReduce): """Shared base class for DataArray and Variable. """ + def __bool__(self: Any) -> bool: return bool(self.values) @@ -616,11 +617,40 @@ def rolling(self, dim: Optional[Mapping[Hashable, int]] = None, return self._rolling_cls(self, dim, min_periods=min_periods, center=center) - def rolling_exp(self, dim=None, window_type='span', **dim_kwargs): + def rolling_exp( + self, + window: Optional[Mapping[str, int]] = None, + window_type: str = 'span', + **dim_kwargs + ): + """ + Exponentially-weighted moving window. + Similar to EWM in pandas + + Parameters + ---------- + window : A single mapping from a dimension name to window value, + optional + dim : str + Name of the dimension to create the rolling exponential window + along (e.g., `time`). + window : int + Size of the moving window. The type of this is specified in + `window_type` + window_type : str, one of ['span', 'com', 'halflife', 'alpha'], + default 'span' + The format of the previously supplied window. Each is a simple + numerical transformation of the others. Described in detail: + https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.ewm.html + **dim_kwargs : optional + The keyword arguments form of ``dim``. + One of dim or dim_kwargs must be provided. - dim = either_dict_or_kwargs(dim, dim_kwargs, 'rolling_exp') + [docstring copied from RollingExp object] + """ + window = either_dict_or_kwargs(window, dim_kwargs, 'rolling_exp') - return self._rolling_exp_cls(self, dim, window_type) + return self._rolling_exp_cls(self, window, window_type) def coarsen(self, dim: Optional[Mapping[Hashable, int]] = None, boundary: str = 'exact', diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 4a78564e86d..3f07babb2f1 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -18,7 +18,7 @@ def move_exp_nanmean(array, *, axis, alpha): if axis == (): return array.astype(np.float64) else: - return numbagg.moving.move_exp_nanmean( + return numbagg.move_exp_nanmean( array, axis=axis, alpha=alpha) @@ -66,7 +66,7 @@ class RollingExp(object): ---------- obj : Dataset or DataArray Object to window. - windows : A mapping from a dimension name to window value + windows : A single mapping from a single dimension name to window value dim : str Name of the dimension to create the rolling exponential window along (e.g., `time`). From 8b95388230c73fba3ee66cf32cc35de8773ff5f4 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 21 Jun 2019 00:55:24 -0400 Subject: [PATCH 28/32] mypy --- setup.cfg | 2 ++ xarray/core/common.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index cdfe2ec3e36..5929116c323 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,8 @@ ignore_missing_imports = True ignore_missing_imports = True [mypy-nc_time_axis.*] ignore_missing_imports = True +[mypy-numbagg.*] +ignore_missing_imports = True [mypy-numpy.*] ignore_missing_imports = True [mypy-netCDF4.*] diff --git a/xarray/core/common.py b/xarray/core/common.py index c589de403e0..7c1af3a31cd 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -619,7 +619,7 @@ def rolling(self, dim: Optional[Mapping[Hashable, int]] = None, def rolling_exp( self, - window: Optional[Mapping[str, int]] = None, + window: Optional[Mapping[Hashable, int]] = None, window_type: str = 'span', **dim_kwargs ): From f3fc3f770e2935b0db6e241a18fe746a987d80cd Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 21 Jun 2019 00:55:42 -0400 Subject: [PATCH 29/32] flake --- xarray/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index 7c1af3a31cd..4cb4991f949 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -629,7 +629,7 @@ def rolling_exp( Parameters ---------- - window : A single mapping from a dimension name to window value, + window : A single mapping from a dimension name to window value, optional dim : str Name of the dimension to create the rolling exponential window From efdbd1f0242cfbd5086cf8f2e070e7729a2de33c Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 21 Jun 2019 02:02:55 -0400 Subject: [PATCH 30/32] pytest config tiny tweak while I'm here --- doc/api.rst | 2 +- setup.cfg | 9 ++++----- xarray/core/common.py | 22 +++++++++++----------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 9ec1ad48efd..811e3241438 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -537,7 +537,7 @@ Rolling objects core.rolling.DatasetRolling core.rolling.DatasetRolling.construct core.rolling.DatasetRolling.reduce - core.rolling_exp.DatasetRollingExp + core.rolling_exp.RollingExp Resample objects ================ diff --git a/setup.cfg b/setup.cfg index 5929116c323..bfa49118d84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,8 +9,11 @@ filterwarnings = ignore:Using a non-tuple sequence for multidimensional indexing is deprecated:FutureWarning env = UVCDAT_ANONYMOUS_LOG=no +markers = + flaky: flaky tests + network: tests requiring a network connection + slow: slow tests -# This should be kept in sync with .pep8speaks.yml [flake8] max-line-length=79 ignore= @@ -23,10 +26,6 @@ ignore= F401 exclude= doc -markers = - flaky: flaky tests - network: tests requiring a network connection - slow: slow tests [isort] default_section=THIRDPARTY diff --git a/xarray/core/common.py b/xarray/core/common.py index 4cb4991f949..b197065e4da 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -557,7 +557,7 @@ def groupby_bins(self, group, bins, right: bool = True, labels=None, def rolling(self, dim: Optional[Mapping[Hashable, int]] = None, min_periods: Optional[int] = None, center: bool = False, - **dim_kwargs: int): + **window_kwargs: int): """ Rolling window object. @@ -572,9 +572,9 @@ def rolling(self, dim: Optional[Mapping[Hashable, int]] = None, setting min_periods equal to the size of the window. center : boolean, default False Set the labels at the center of the window. - **dim_kwargs : optional + **window_kwargs : optional The keyword arguments form of ``dim``. - One of dim or dim_kwargs must be provided. + One of dim or window_kwargs must be provided. Returns ------- @@ -613,7 +613,7 @@ def rolling(self, dim: Optional[Mapping[Hashable, int]] = None, core.rolling.DataArrayRolling core.rolling.DatasetRolling """ # noqa - dim = either_dict_or_kwargs(dim, dim_kwargs, 'rolling') + dim = either_dict_or_kwargs(dim, window_kwargs, 'rolling') return self._rolling_cls(self, dim, min_periods=min_periods, center=center) @@ -621,7 +621,7 @@ def rolling_exp( self, window: Optional[Mapping[Hashable, int]] = None, window_type: str = 'span', - **dim_kwargs + **window_kwargs ): """ Exponentially-weighted moving window. @@ -642,13 +642,13 @@ def rolling_exp( The format of the previously supplied window. Each is a simple numerical transformation of the others. Described in detail: https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.ewm.html - **dim_kwargs : optional - The keyword arguments form of ``dim``. - One of dim or dim_kwargs must be provided. + **window_kwargs : optional + The keyword arguments form of ``window``. + One of window or window_kwargs must be provided. [docstring copied from RollingExp object] """ - window = either_dict_or_kwargs(window, dim_kwargs, 'rolling_exp') + window = either_dict_or_kwargs(window, window_kwargs, 'rolling_exp') return self._rolling_exp_cls(self, window, window_type) @@ -656,7 +656,7 @@ def coarsen(self, dim: Optional[Mapping[Hashable, int]] = None, boundary: str = 'exact', side: Union[str, Mapping[Hashable, str]] = 'left', coord_func: str = 'mean', - **dim_kwargs: int): + **window_kwargs: int): """ Coarsen object. @@ -710,7 +710,7 @@ def coarsen(self, dim: Optional[Mapping[Hashable, int]] = None, core.rolling.DataArrayCoarsen core.rolling.DatasetCoarsen """ - dim = either_dict_or_kwargs(dim, dim_kwargs, 'coarsen') + dim = either_dict_or_kwargs(dim, window_kwargs, 'coarsen') return self._coarsen_cls( self, dim, boundary=boundary, side=side, coord_func=coord_func) From 7b094bae4e67d8f8f7022bf6510a4d71e1cfca92 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 23 Jun 2019 14:26:58 +0300 Subject: [PATCH 31/32] Rolling exp doc updates --- doc/installing.rst | 2 ++ xarray/core/common.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/installing.rst b/doc/installing.rst index f624da18611..b9d1b4d0ba4 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -45,6 +45,8 @@ For accelerating xarray - `bottleneck `__: speeds up NaN-skipping and rolling window aggregations by a large factor (1.1 or later) +- `numbagg `_: for exponential rolling + window operations For parallel computing ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/xarray/core/common.py b/xarray/core/common.py index b197065e4da..0195be62500 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -627,6 +627,8 @@ def rolling_exp( Exponentially-weighted moving window. Similar to EWM in pandas + Requires the optional Numbagg dependency. + Parameters ---------- window : A single mapping from a dimension name to window value, @@ -646,7 +648,9 @@ def rolling_exp( The keyword arguments form of ``window``. One of window or window_kwargs must be provided. - [docstring copied from RollingExp object] + See Also + -------- + core.rolling_exp.RollingExp """ window = either_dict_or_kwargs(window, window_kwargs, 'rolling_exp') From dd2a791d78335d4df5d21feaa53b378019c38f37 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sun, 23 Jun 2019 23:44:37 -0400 Subject: [PATCH 32/32] remove _attributes from RollingExp class --- xarray/core/rolling_exp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xarray/core/rolling_exp.py b/xarray/core/rolling_exp.py index 3f07babb2f1..ff6baef5c3a 100644 --- a/xarray/core/rolling_exp.py +++ b/xarray/core/rolling_exp.py @@ -57,7 +57,7 @@ def _get_center_of_mass(comass, span, halflife, alpha): return float(comass) -class RollingExp(object): +class RollingExp: """ Exponentially-weighted moving window object. Similar to EWM in pandas @@ -82,7 +82,6 @@ class RollingExp(object): ------- RollingExp : type of input argument """ # noqa - _attributes = ['alpha', 'dim'] def __init__(self, obj, windows, window_type='span'): self.obj = obj