Merge branch 'master' into kms/drop_dims

Author: kmsquire

doc/whats-new.rst

@@ -68,9 +68,17 @@ Enhancements
 - :py:meth:`pandas.Series.dropna` is now supported for a
   :py:class:`pandas.Series` indexed by a :py:class:`~xarray.CFTimeIndex`
   (:issue:`2688`). By `Spencer Clark <https://github.com/spencerkclark>`_.
+- :py:meth:`~xarray.open_dataset` now accepts a ``use_cftime`` argument, which
+  can be used to require that ``cftime.datetime`` objects are always used, or
+  never used when decoding dates encoded with a standard calendar.  This can be
+  used to ensure consistent date types are returned when using
+  :py:meth:`~xarray.open_mfdataset` (:issue:`1263`) and/or to silence
+  serialization warnings raised if dates from a standard calendar are found to
+  be outside the :py:class:`pandas.Timestamp`-valid range (:issue:`2754`).  By
+  `Spencer Clark <https://github.com/spencerkclark>`_. 
 - Added :py:meth:`~xarray.Dataset.drop_dims` (:issue:`1949`).
   By `Kevin Squire <https://github.com/kmsquire>`_.
-
+  
 Bug fixes
 ~~~~~~~~~
 
@@ -96,6 +104,16 @@ Bug fixes
 - Masking data arrays with :py:meth:`xarray.DataArray.where` now returns an
   array with the name of the original masked array (:issue:`2748` and :issue:`2457`).
   By `Yohai Bar-Sinai <https://github.com/yohai>`_.
+- Fixed error when trying to reduce a DataArray using a function which does not
+  require an axis argument. (:issue:`2768`)
+  By `Tom Nicholas <http://github.com/TomNicholas>`_.
+
+- Per `CF conventions
+  <http://cfconventions.org/cf-conventions/cf-conventions.html#calendar>`_,
+  specifying ``'standard'`` as the calendar type in
+  :py:meth:`~xarray.cftime_range` now correctly refers to the ``'gregorian'``
+  calendar instead of the ``'proleptic_gregorian'`` calendar (:issue:`2761`).
+
 .. _whats-new.0.11.3:
 
 v0.11.3 (26 January 2019)

xarray/backends/api.py

@@ -161,7 +161,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True,
                  mask_and_scale=None, decode_times=True, autoclose=None,
                  concat_characters=True, decode_coords=True, engine=None,
                  chunks=None, lock=None, cache=None, drop_variables=None,
-                 backend_kwargs=None):
+                 backend_kwargs=None, use_cftime=None):
     """Load and decode a dataset from a file or file-like object.
 
     Parameters
@@ -231,6 +231,16 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True,
         A dictionary of keyword arguments to pass on to the backend. This
         may be useful when backend options would improve performance or
         allow user control of dataset processing.
+    use_cftime: bool, optional
+        Only relevant if encoded dates come from a standard calendar
+        (e.g. 'gregorian', 'proleptic_gregorian', 'standard', or not
+        specified).  If None (default), attempt to decode times to
+        ``np.datetime64[ns]`` objects; if this is not possible, decode times to
+        ``cftime.datetime`` objects. If True, always decode times to
+        ``cftime.datetime`` objects, regardless of whether or not they can be
+        represented using ``np.datetime64[ns]`` objects.  If False, always
+        decode times to ``np.datetime64[ns]`` objects; if this is not possible
+        raise an error.
 
     Returns
     -------
@@ -269,7 +279,7 @@ def maybe_decode_store(store, lock=False):
         ds = conventions.decode_cf(
             store, mask_and_scale=mask_and_scale, decode_times=decode_times,
             concat_characters=concat_characters, decode_coords=decode_coords,
-            drop_variables=drop_variables)
+            drop_variables=drop_variables, use_cftime=use_cftime)
 
         _protect_dataset_variables_inplace(ds, cache)
 
@@ -284,7 +294,8 @@ def maybe_decode_store(store, lock=False):
                 mtime = None
             token = tokenize(filename_or_obj, mtime, group, decode_cf,
                              mask_and_scale, decode_times, concat_characters,
-                             decode_coords, engine, chunks, drop_variables)
+                             decode_coords, engine, chunks, drop_variables,
+                             use_cftime)
             name_prefix = 'open_dataset-%s' % token
             ds2 = ds.chunk(chunks, name_prefix=name_prefix, token=token)
             ds2._file_obj = ds._file_obj
@@ -360,7 +371,7 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True,
                    mask_and_scale=None, decode_times=True, autoclose=None,
                    concat_characters=True, decode_coords=True, engine=None,
                    chunks=None, lock=None, cache=None, drop_variables=None,
-                   backend_kwargs=None):
+                   backend_kwargs=None, use_cftime=None):
     """Open an DataArray from a netCDF file containing a single data variable.
 
     This is designed to read netCDF files with only one data variable. If
@@ -428,6 +439,16 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True,
         A dictionary of keyword arguments to pass on to the backend. This
         may be useful when backend options would improve performance or
         allow user control of dataset processing.
+    use_cftime: bool, optional
+        Only relevant if encoded dates come from a standard calendar
+        (e.g. 'gregorian', 'proleptic_gregorian', 'standard', or not
+        specified).  If None (default), attempt to decode times to
+        ``np.datetime64[ns]`` objects; if this is not possible, decode times to
+        ``cftime.datetime`` objects. If True, always decode times to
+        ``cftime.datetime`` objects, regardless of whether or not they can be
+        represented using ``np.datetime64[ns]`` objects.  If False, always
+        decode times to ``np.datetime64[ns]`` objects; if this is not possible
+        raise an error.
 
     Notes
     -----
@@ -450,7 +471,8 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True,
                            decode_coords=decode_coords, engine=engine,
                            chunks=chunks, lock=lock, cache=cache,
                            drop_variables=drop_variables,
-                           backend_kwargs=backend_kwargs)
+                           backend_kwargs=backend_kwargs,
+                           use_cftime=use_cftime)
 
     if len(dataset.data_vars) != 1:
         raise ValueError('Given file dataset contains more than one data '

xarray/coding/cftime_offsets.py

@@ -68,7 +68,7 @@ def get_date_type(calendar):
             'proleptic_gregorian': cftime.DatetimeProlepticGregorian,
             'julian': cftime.DatetimeJulian,
             'all_leap': cftime.DatetimeAllLeap,
-            'standard': cftime.DatetimeProlepticGregorian
+            'standard': cftime.DatetimeGregorian
         }
         return calendars[calendar]
 
@@ -679,9 +679,9 @@ def cftime_range(start=None, end=None, periods=None, freq='D',
     +--------------------------------+---------------------------------------+
     | Alias                          | Date type                             |
     +================================+=======================================+
-    | standard, proleptic_gregorian  | ``cftime.DatetimeProlepticGregorian`` |
+    | standard, gregorian            | ``cftime.DatetimeGregorian``          |
     +--------------------------------+---------------------------------------+
-    | gregorian                      | ``cftime.DatetimeGregorian``          |
+    | proleptic_gregorian            | ``cftime.DatetimeProlepticGregorian`` |
     +--------------------------------+---------------------------------------+
     | noleap, 365_day                | ``cftime.DatetimeNoLeap``             |
     +--------------------------------+---------------------------------------+

xarray/coding/times.py

@@ -80,32 +80,7 @@ def _unpack_netcdf_time_units(units):
     return delta_units, ref_date
 
 
-def _decode_datetime_with_cftime(num_dates, units, calendar):
-    cftime = _import_cftime()
-
-    if cftime.__name__ == 'cftime':
-        dates = np.asarray(cftime.num2date(num_dates, units, calendar,
-                                           only_use_cftime_datetimes=True))
-    else:
-        # Must be using num2date from an old version of netCDF4 which
-        # does not have the only_use_cftime_datetimes option.
-        dates = np.asarray(cftime.num2date(num_dates, units, calendar))
-
-    if (dates[np.nanargmin(num_dates)].year < 1678 or
-            dates[np.nanargmax(num_dates)].year >= 2262):
-        if calendar in _STANDARD_CALENDARS:
-            warnings.warn(
-                'Unable to decode time axis into full '
-                'numpy.datetime64 objects, continuing using dummy '
-                'cftime.datetime objects instead, reason: dates out '
-                'of range', SerializationWarning, stacklevel=3)
-    else:
-        if calendar in _STANDARD_CALENDARS:
-            dates = cftime_to_nptime(dates)
-    return dates
-
-
-def _decode_cf_datetime_dtype(data, units, calendar):
+def _decode_cf_datetime_dtype(data, units, calendar, use_cftime):
     # Verify that at least the first and last date can be decoded
     # successfully. Otherwise, tracebacks end up swallowed by
     # Dataset.__repr__ when users try to view their lazily decoded array.
@@ -115,7 +90,8 @@ def _decode_cf_datetime_dtype(data, units, calendar):
                                     last_item(values) or [0]])
 
     try:
-        result = decode_cf_datetime(example_value, units, calendar)
+        result = decode_cf_datetime(example_value, units, calendar,
+                                    use_cftime)
     except Exception:
         calendar_msg = ('the default calendar' if calendar is None
                         else 'calendar %r' % calendar)
@@ -129,7 +105,52 @@ def _decode_cf_datetime_dtype(data, units, calendar):
     return dtype
 
 
-def decode_cf_datetime(num_dates, units, calendar=None):
+def _decode_datetime_with_cftime(num_dates, units, calendar):
+    cftime = _import_cftime()
+
+    if cftime.__name__ == 'cftime':
+        return np.asarray(cftime.num2date(num_dates, units, calendar,
+                                          only_use_cftime_datetimes=True))
+    else:
+        # Must be using num2date from an old version of netCDF4 which
+        # does not have the only_use_cftime_datetimes option.
+        return np.asarray(cftime.num2date(num_dates, units, calendar))
+
+
+def _decode_datetime_with_pandas(flat_num_dates, units, calendar):
+    if calendar not in _STANDARD_CALENDARS:
+        raise OutOfBoundsDatetime(
+            'Cannot decode times from a non-standard calendar, {!r}, using '
+            'pandas.'.format(calendar))
+
+    delta, ref_date = _unpack_netcdf_time_units(units)
+    delta = _netcdf_to_numpy_timeunit(delta)
+    try:
+        ref_date = pd.Timestamp(ref_date)
+    except ValueError:
+        # ValueError is raised by pd.Timestamp for non-ISO timestamp
+        # strings, in which case we fall back to using cftime
+        raise OutOfBoundsDatetime
+
+    # fixes: https://github.com/pydata/pandas/issues/14068
+    # these lines check if the the lowest or the highest value in dates
+    # cause an OutOfBoundsDatetime (Overflow) error
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', 'invalid value encountered',
+                                RuntimeWarning)
+        pd.to_timedelta(flat_num_dates.min(), delta) + ref_date
+        pd.to_timedelta(flat_num_dates.max(), delta) + ref_date
+
+    # Cast input dates to integers of nanoseconds because `pd.to_datetime`
+    # works much faster when dealing with integers
+    # make _NS_PER_TIME_DELTA an array to ensure type upcasting
+    flat_num_dates_ns_int = (flat_num_dates.astype(np.float64) *
+                             _NS_PER_TIME_DELTA[delta]).astype(np.int64)
+
+    return (pd.to_timedelta(flat_num_dates_ns_int, 'ns') + ref_date).values
+
+
+def decode_cf_datetime(num_dates, units, calendar=None, use_cftime=None):
     """Given an array of numeric dates in netCDF format, convert it into a
     numpy array of date time objects.
 
@@ -149,41 +170,30 @@ def decode_cf_datetime(num_dates, units, calendar=None):
     if calendar is None:
         calendar = 'standard'
 
-    delta, ref_date = _unpack_netcdf_time_units(units)
-
-    try:
-        if calendar not in _STANDARD_CALENDARS:
-            raise OutOfBoundsDatetime
-
-        delta = _netcdf_to_numpy_timeunit(delta)
+    if use_cftime is None:
         try:
-            ref_date = pd.Timestamp(ref_date)
-        except ValueError:
-            # ValueError is raised by pd.Timestamp for non-ISO timestamp
-            # strings, in which case we fall back to using cftime
-            raise OutOfBoundsDatetime
-
-        # fixes: https://github.com/pydata/pandas/issues/14068
-        # these lines check if the the lowest or the highest value in dates
-        # cause an OutOfBoundsDatetime (Overflow) error
-        with warnings.catch_warnings():
-            warnings.filterwarnings('ignore', 'invalid value encountered',
-                                    RuntimeWarning)
-            pd.to_timedelta(flat_num_dates.min(), delta) + ref_date
-            pd.to_timedelta(flat_num_dates.max(), delta) + ref_date
-
-        # Cast input dates to integers of nanoseconds because `pd.to_datetime`
-        # works much faster when dealing with integers
-        # make _NS_PER_TIME_DELTA an array to ensure type upcasting
-        flat_num_dates_ns_int = (flat_num_dates.astype(np.float64) *
-                                 _NS_PER_TIME_DELTA[delta]).astype(np.int64)
-
-        dates = (pd.to_timedelta(flat_num_dates_ns_int, 'ns') +
-                 ref_date).values
-
-    except (OutOfBoundsDatetime, OverflowError):
+            dates = _decode_datetime_with_pandas(flat_num_dates, units,
+                                                 calendar)
+        except (OutOfBoundsDatetime, OverflowError):
+            dates = _decode_datetime_with_cftime(
+                flat_num_dates.astype(np.float), units, calendar)
+
+            if (dates[np.nanargmin(num_dates)].year < 1678 or
+               dates[np.nanargmax(num_dates)].year >= 2262):
+                if calendar in _STANDARD_CALENDARS:
+                    warnings.warn(
+                        'Unable to decode time axis into full '
+                        'numpy.datetime64 objects, continuing using '
+                        'cftime.datetime objects instead, reason: dates out '
+                        'of range', SerializationWarning, stacklevel=3)
+            else:
+                if calendar in _STANDARD_CALENDARS:
+                    dates = cftime_to_nptime(dates)
+    elif use_cftime:
         dates = _decode_datetime_with_cftime(
             flat_num_dates.astype(np.float), units, calendar)
+    else:
+        dates = _decode_datetime_with_pandas(flat_num_dates, units, calendar)
 
     return dates.reshape(num_dates.shape)
 
@@ -383,6 +393,8 @@ def encode_cf_timedelta(timedeltas, units=None):
 
 
 class CFDatetimeCoder(VariableCoder):
+    def __init__(self, use_cftime=None):
+        self.use_cftime = use_cftime
 
     def encode(self, variable, name=None):
         dims, data, attrs, encoding = unpack_for_encoding(variable)
@@ -403,9 +415,11 @@ def decode(self, variable, name=None):
         if 'units' in attrs and 'since' in attrs['units']:
             units = pop_to(attrs, encoding, 'units')
             calendar = pop_to(attrs, encoding, 'calendar')
-            dtype = _decode_cf_datetime_dtype(data, units, calendar)
+            dtype = _decode_cf_datetime_dtype(data, units, calendar,
+                                              self.use_cftime)
             transform = partial(
-                decode_cf_datetime, units=units, calendar=calendar)
+                decode_cf_datetime, units=units, calendar=calendar,
+                use_cftime=self.use_cftime)
             data = lazy_elemwise_func(data, transform, dtype)
 
         return Variable(dims, data, attrs, encoding)