Merge remote-tracking branch 'upstream/master' into fix/2932

dcherian · dcherian · commit 106034db304c · 2019-05-06T11:59:48.000-06:00
* upstream/master: [WIP] Custom fill value for reindex, align, and merge operations (pydata#2920) Attempt to fix py35 build on Travis (pydata#2925) List formatting in docs (pydata#2939) DOC: Avoid downloading .tif file (pydata#2919)
diff --git a/.travis.yml b/.travis.yml
@@ -11,7 +11,6 @@ matrix:
   fast_finish: true
   include:
   - env: CONDA_ENV=py35-min
-  - env: CONDA_ENV=py35
   - env: CONDA_ENV=py36
   - env: CONDA_ENV=py37
   - env:
diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml
diff --git a/ci/requirements-py36-dask-dev.yml b/ci/requirements-py36-dask-dev.yml
@@ -13,8 +13,8 @@ dependencies:
   - pytest-env
   - coveralls
   - flake8
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
diff --git a/ci/requirements-py36-hypothesis.yml b/ci/requirements-py36-hypothesis.yml
@@ -15,8 +15,8 @@ dependencies:
   - coveralls
   - hypothesis
   - flake8
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
diff --git a/ci/requirements-py36-pandas-dev.yml b/ci/requirements-py36-pandas-dev.yml
@@ -16,7 +16,7 @@ dependencies:
   - pytest-env
   - coveralls
   - flake8
-  - numpy
+  - numpy>=1.12
   - scipy
   - toolz
   - pip:
diff --git a/ci/requirements-py36-rasterio.yml b/ci/requirements-py36-rasterio.yml
@@ -14,8 +14,8 @@ dependencies:
   - pytest-cov
   - pytest-env
   - coveralls
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
diff --git a/ci/requirements-py36-windows.yml b/ci/requirements-py36-windows.yml
@@ -12,11 +12,10 @@ dependencies:
   - netcdf4
   - pytest
   - pytest-env
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
   - rasterio
   - zarr
-
diff --git a/ci/requirements-py36-zarr-dev.yml b/ci/requirements-py36-zarr-dev.yml
@@ -12,8 +12,8 @@ dependencies:
   - pytest-env
   - coveralls
   - flake8
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml
@@ -15,8 +15,8 @@ dependencies:
   - pytest-env
   - coveralls
   - pycodestyle
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
diff --git a/ci/requirements-py37-windows.yml b/ci/requirements-py37-windows.yml
@@ -13,8 +13,8 @@ dependencies:
   - netcdf4
   - pytest
   - pytest-env
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml
@@ -16,8 +16,8 @@ dependencies:
   - pytest-env
   - coveralls
   - pycodestyle
-  - numpy
-  - pandas
+  - numpy>=1.12
+  - pandas>=0.19
   - scipy
   - seaborn
   - toolz
diff --git a/doc/gallery/plot_rasterio.py b/doc/gallery/plot_rasterio.py
@@ -16,22 +16,16 @@
 original map projection (see :ref:`recipes.rasterio_rgb`).
 """
 
-import os
-import urllib.request
-
 import cartopy.crs as ccrs
 import matplotlib.pyplot as plt
 import numpy as np
 from rasterio.warp import transform
 
 import xarray as xr
 
-# Download the file from rasterio's repository
-url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif'
-urllib.request.urlretrieve(url, 'RGB.byte.tif')
-
 # Read the data
-da = xr.open_rasterio('RGB.byte.tif')
+url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif'
+da = xr.open_rasterio(url)
 
 # Compute the lon/lat coordinates with rasterio.warp.transform
 ny, nx = len(da['y']), len(da['x'])
@@ -54,6 +48,3 @@
                cmap='Greys_r', add_colorbar=False)
 ax.coastlines('10m', color='r')
 plt.show()
-
-# Delete the file
-os.remove('RGB.byte.tif')
diff --git a/doc/gallery/plot_rasterio_rgb.py b/doc/gallery/plot_rasterio_rgb.py
@@ -13,20 +13,14 @@
 transformation.
 """
 
-import os
-import urllib.request
-
 import cartopy.crs as ccrs
 import matplotlib.pyplot as plt
 
 import xarray as xr
 
-# Download the file from rasterio's repository
-url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif'
-urllib.request.urlretrieve(url, 'RGB.byte.tif')
-
 # Read the data
-da = xr.open_rasterio('RGB.byte.tif')
+url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif'
+da = xr.open_rasterio(url)
 
 # The data is in UTM projection. We have to set it manually until
 # https://github.com/SciTools/cartopy/issues/813 is implemented
@@ -37,6 +31,3 @@
 da.plot.imshow(ax=ax, rgb='band', transform=crs)
 ax.coastlines('10m', color='r')
 plt.show()
-
-# Delete the file
-os.remove('RGB.byte.tif')
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
@@ -21,6 +21,9 @@ v0.12.2 (unreleased)
 Enhancements
 ~~~~~~~~~~~~
 
+- Add ``fill_value`` argument for reindex, align, and merge operations
+  to enable custom fill values. (:issue:`2876`)
+  By `Zach Griffith <https://github.com/zdgriffith>`_.
 - Character arrays' character dimension name decoding and encoding handled by
   ``var.encoding['char_dim_name']`` (:issue:`2895`)
   By `James McCreight <https://github.com/jmccreight>`_.
diff --git a/xarray/core/alignment.py b/xarray/core/alignment.py
@@ -8,7 +8,7 @@
 import numpy as np
 import pandas as pd
 
-from . import utils
+from . import utils, dtypes
 from .indexing import get_indexer_nd
 from .utils import is_dict_like, is_full_slice
 from .variable import IndexVariable, Variable
@@ -31,20 +31,17 @@ def _get_joiner(join):
         raise ValueError('invalid value for join: %s' % join)
 
 
-_DEFAULT_EXCLUDE = frozenset()  # type: frozenset
-
-
-def align(*objects, **kwargs):
-    """align(*objects, join='inner', copy=True, indexes=None,
-             exclude=frozenset())
-
+def align(*objects, join='inner', copy=True, indexes=None, exclude=frozenset(),
+          fill_value=dtypes.NA):
+    """
     Given any number of Dataset and/or DataArray objects, returns new
     objects with aligned indexes and dimension sizes.
 
     Array from the aligned objects are suitable as input to mathematical
     operators, because along each dimension they have the same index and size.
 
-    Missing values (if ``join != 'inner'``) are filled with NaN.
+    Missing values (if ``join != 'inner'``) are filled with ``fill_value``.
+    The default fill value is NaN.
 
     Parameters
     ----------
@@ -65,11 +62,13 @@ def align(*objects, **kwargs):
         ``copy=False`` and reindexing is unnecessary, or can be performed with
         only slice operations, then the output may share memory with the input.
         In either case, new xarray objects are always returned.
-    exclude : sequence of str, optional
-        Dimensions that must be excluded from alignment
     indexes : dict-like, optional
         Any indexes explicitly provided with the `indexes` argument should be
         used in preference to the aligned indexes.
+    exclude : sequence of str, optional
+        Dimensions that must be excluded from alignment
+    fill_value : scalar, optional
+        Value to use for newly missing values
 
     Returns
     -------
@@ -82,15 +81,8 @@ def align(*objects, **kwargs):
         If any dimensions without labels on the arguments have different sizes,
         or a different size than the size of the aligned dimension labels.
     """
-    join = kwargs.pop('join', 'inner')
-    copy = kwargs.pop('copy', True)
-    indexes = kwargs.pop('indexes', None)
-    exclude = kwargs.pop('exclude', _DEFAULT_EXCLUDE)
     if indexes is None:
         indexes = {}
-    if kwargs:
-        raise TypeError('align() got unexpected keyword arguments: %s'
-                        % list(kwargs))
 
     if not indexes and len(objects) == 1:
         # fast path for the trivial case
@@ -162,15 +154,17 @@ def align(*objects, **kwargs):
             # fast path for no reindexing necessary
             new_obj = obj.copy(deep=copy)
         else:
-            new_obj = obj.reindex(copy=copy, **valid_indexers)
+            new_obj = obj.reindex(copy=copy, fill_value=fill_value,
+                                  **valid_indexers)
         new_obj.encoding = obj.encoding
         result.append(new_obj)
 
     return tuple(result)
 
 
 def deep_align(objects, join='inner', copy=True, indexes=None,
-               exclude=frozenset(), raise_on_invalid=True):
+               exclude=frozenset(), raise_on_invalid=True,
+               fill_value=dtypes.NA):
     """Align objects for merging, recursing into dictionary values.
 
     This function is not public API.
@@ -214,7 +208,7 @@ def is_alignable(obj):
             out.append(variables)
 
     aligned = align(*targets, join=join, copy=copy, indexes=indexes,
-                    exclude=exclude)
+                    exclude=exclude, fill_value=fill_value)
 
     for position, key, aligned_obj in zip(positions, keys, aligned):
         if key is no_key:
@@ -270,6 +264,7 @@ def reindex_variables(
     method: Optional[str] = None,
     tolerance: Any = None,
     copy: bool = True,
+    fill_value: Optional[Any] = dtypes.NA,
 ) -> 'Tuple[OrderedDict[Any, Variable], OrderedDict[Any, pd.Index]]':
     """Conform a dictionary of aligned variables onto a new set of variables,
     filling in missing values with NaN.
@@ -305,6 +300,8 @@ def reindex_variables(
         ``copy=False`` and reindexing is unnecessary, or can be performed
         with only slice operations, then the output may share memory with
         the input. In either case, new xarray objects are always returned.
+    fill_value : scalar, optional
+        Value to use for newly missing values
 
     Returns
     -------
@@ -380,7 +377,7 @@ def reindex_variables(
             needs_masking = any(d in masked_dims for d in var.dims)
 
             if needs_masking:
-                new_var = var._getitem_with_mask(key)
+                new_var = var._getitem_with_mask(key, fill_value=fill_value)
             elif all(is_full_slice(k) for k in key):
                 # no reindexing necessary
                 # here we need to manually deal with copying data, since
diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py
@@ -879,9 +879,10 @@ def sel_points(self, dim='points', method=None, tolerance=None,
             dim=dim, method=method, tolerance=tolerance, **indexers)
         return self._from_temp_dataset(ds)
 
-    def reindex_like(self, other, method=None, tolerance=None, copy=True):
-        """Conform this object onto the indexes of another object, filling
-        in missing values with NaN.
+    def reindex_like(self, other, method=None, tolerance=None, copy=True,
+                     fill_value=dtypes.NA):
+        """Conform this object onto the indexes of another object, filling in
+        missing values with ``fill_value``. The default fill value is NaN.
 
         Parameters
         ----------
@@ -910,6 +911,8 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True):
             ``copy=False`` and reindexing is unnecessary, or can be performed
             with only slice operations, then the output may share memory with
             the input. In either case, a new xarray object is always returned.
+        fill_value : scalar, optional
+            Value to use for newly missing values
 
         Returns
         -------
@@ -924,12 +927,12 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True):
         """
         indexers = reindex_like_indexers(self, other)
         return self.reindex(method=method, tolerance=tolerance, copy=copy,
-                            **indexers)
+                            fill_value=fill_value, **indexers)
 
     def reindex(self, indexers=None, method=None, tolerance=None, copy=True,
-                **indexers_kwargs):
-        """Conform this object onto a new set of indexes, filling in
-        missing values with NaN.
+                fill_value=dtypes.NA, **indexers_kwargs):
+        """Conform this object onto the indexes of another object, filling in
+        missing values with ``fill_value``. The default fill value is NaN.
 
         Parameters
         ----------
@@ -956,6 +959,8 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True,
             Maximum distance between original and new labels for inexact
             matches. The values of the index at the matching locations must
             satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+        fill_value : scalar, optional
+            Value to use for newly missing values
         **indexers_kwarg : {dim: indexer, ...}, optional
             The keyword arguments form of ``indexers``.
             One of indexers or indexers_kwargs must be provided.
@@ -974,7 +979,8 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True,
         indexers = either_dict_or_kwargs(
             indexers, indexers_kwargs, 'reindex')
         ds = self._to_temp_dataset().reindex(
-            indexers=indexers, method=method, tolerance=tolerance, copy=copy)
+            indexers=indexers, method=method, tolerance=tolerance, copy=copy,
+            fill_value=fill_value)
         return self._from_temp_dataset(ds)
 
     def interp(self, coords=None, method='linear', assume_sorted=False,
diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py
diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py
diff --git a/xarray/core/merge.py b/xarray/core/merge.py
diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py
diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py
diff --git a/xarray/tests/test_merge.py b/xarray/tests/test_merge.py
