Dataset.encoding and unlimited dimensions for to_netcdf

doc/api.rst

@@ -46,6 +46,7 @@ Attributes
    Dataset.data_vars
    Dataset.coords
    Dataset.attrs
+   Dataset.encoding
    Dataset.indexes
    Dataset.get_index
 

doc/whats-new.rst

@@ -166,6 +166,10 @@ Enhancements
   similar to what the command line utility ``ncdump -h`` produces (:issue:`1150`).
   By `Joe Hamman <https://github.com/jhamman>`_.
 
+- Added the ability write unlimited netCDF dimensions with the ``netcdf4``
+  backend.
+  By `Joe Hamman <https://github.com/jhamman>`_.
+
 Bug fixes
 ~~~~~~~~~
 - ``groupby_bins`` now restores empty bins by default (:issue:`1019`).

xarray/backends/api.py

@@ -565,6 +565,8 @@ def to_netcdf(dataset, path=None, mode='w', format=None, group=None,
     sync = writer is None
 
     store = store_cls(path, mode, format, group, writer)
+    # Copy dataset encoding to datastore
+    store.encoding = dataset.encoding
     try:
         dataset.dump_to_store(store, sync=sync, encoding=encoding)
         if isinstance(path, BytesIO):

xarray/backends/common.py

@@ -2,7 +2,6 @@
 from __future__ import division
 from __future__ import print_function
 import numpy as np
-import itertools
 import logging
 import time
 import traceback
@@ -12,7 +11,7 @@
 
 from ..conventions import cf_encoder
 from ..core.utils import FrozenOrderedDict
-from ..core.pycompat import iteritems, dask_array_type, OrderedDict
+from ..core.pycompat import iteritems, dask_array_type
 
 # Create a logger object, but don't add any handlers. Leave that to user code.
 logger = logging.getLogger(__name__)
@@ -75,6 +74,9 @@ def get_attrs(self):  # pragma: no cover
     def get_variables(self):  # pragma: no cover
         raise NotImplementedError
 
+    def get_encoding(self):
+        return {}
+
     def load(self):
         """
         This loads the variables and attributes simultaneously.
@@ -96,8 +98,9 @@ def load(self):
         This function will be called anytime variables or attributes
         are requested, so care should be taken to make sure its fast.
         """
-        variables = FrozenOrderedDict((_decode_variable_name(k), v)
-                                      for k, v in iteritems(self.get_variables()))
+        self.encoding = self.get_encoding()
+        variables = FrozenOrderedDict((_decode_variable_name(k), v) for k, v in
+                                      iteritems(self.get_variables()))
         attributes = FrozenOrderedDict(self.get_attrs())
         return variables, attributes
 
@@ -143,7 +146,11 @@ def add(self, source, target):
             self.sources.append(source)
             self.targets.append(target)
         else:
-            target[...] = source
+            try:
+                target[...] = source
+            except TypeError:
+                # workaround for GH: scipy/scipy#6880
+                target[:] = source
 
     def sync(self):
         if self.sources:
@@ -197,9 +204,11 @@ def set_variables(self, variables, check_encoding_set):
             target, source = self.prepare_variable(name, v, check)
             self.writer.add(source, target)
 
-    def set_necessary_dimensions(self, variable):
+    def set_necessary_dimensions(self, variable, unlimited_dims=set()):
         for d, l in zip(variable.dims, variable.shape):
             if d not in self.dimensions:
+                if d in unlimited_dims:
+                    l = None
                 self.set_dimension(d, l)
 
 

xarray/backends/h5netcdf_.py

@@ -2,15 +2,16 @@
 from __future__ import division
 from __future__ import print_function
 import functools
+import warnings
 
 from .. import Variable
 from ..core import indexing
 from ..core.utils import FrozenOrderedDict, close_on_error, Frozen
 from ..core.pycompat import iteritems, bytes_type, unicode_type, OrderedDict
 
 from .common import WritableCFDataStore, DataStorePickleMixin
-from .netCDF4_ import (_nc4_group, _nc4_values_and_dtype, _extract_nc4_encoding,
-                       BaseNetCDF4Array)
+from .netCDF4_ import (_nc4_group, _nc4_values_and_dtype,
+                       _extract_nc4_variable_encoding, BaseNetCDF4Array)
 
 
 def maybe_decode_bytes(txt):
@@ -33,7 +34,7 @@ def _read_attributes(h5netcdf_var):
     return attrs
 
 
-_extract_h5nc_encoding = functools.partial(_extract_nc4_encoding,
+_extract_h5nc_encoding = functools.partial(_extract_nc4_variable_encoding,
                                            lsd_okay=False, backend='h5netcdf')
 
 
@@ -58,6 +59,7 @@ def __init__(self, filename, mode='r', format=None, group=None,
         self._opener = opener
         self._filename = filename
         self._mode = mode
+        self.encoding = {}
         super(H5NetCDFStore, self).__init__(writer)
 
     def open_store_variable(self, name, var):
@@ -100,7 +102,12 @@ def prepare_variable(self, name, variable, check_encoding=False):
         if dtype is str:
             dtype = h5py.special_dtype(vlen=unicode_type)
 
-        self.set_necessary_dimensions(variable)
+        unlimited_dims = self.encoding.get('unlimited_dims', set())
+        if len(unlimited_dims) > 0:
+            warnings.warn('h5netcdf does not support unlimited dimensions',
+                          UserWarning)
+            unlimited_dims = set()
+        self.set_necessary_dimensions(variable, unlimited_dims=unlimited_dims)
 
         fill_value = attrs.pop('_FillValue', None)
         if fill_value in ['\x00']:

xarray/backends/memory.py

@@ -21,6 +21,7 @@ class InMemoryDataStore(AbstractWritableDataStore):
     def __init__(self, variables=None, attributes=None, writer=None):
         self._variables = OrderedDict() if variables is None else variables
         self._attributes = OrderedDict() if attributes is None else attributes
+        self.encoding = {}
         super(InMemoryDataStore, self).__init__(writer)
 
     def get_attrs(self):

xarray/backends/netCDF4_.py

@@ -7,7 +7,7 @@
 import numpy as np
 
 from .. import Variable
-from ..conventions import pop_to, cf_encoder
+from ..conventions import pop_to
 from ..core import indexing
 from ..core.utils import (FrozenOrderedDict, NDArrayMixin,
                           close_on_error, is_remote_uri)
@@ -138,13 +138,13 @@ def _force_native_endianness(var):
     # check to see if encoding has a value for endian its 'native'
     if not var.encoding.get('endian', 'native') is 'native':
         raise NotImplementedError("Attempt to write non-native endian type, "
-                                  "this is not supported by the netCDF4 python "
-                                  "library.")
+                                  "this is not supported by the netCDF4 "
+                                  "python library.")
     return var
 
 
-def _extract_nc4_encoding(variable, raise_on_invalid=False, lsd_okay=True,
-                          backend='netCDF4'):
+def _extract_nc4_variable_encoding(variable, raise_on_invalid=False,
+                                   lsd_okay=True, backend='netCDF4'):
     encoding = variable.encoding.copy()
 
     safe_to_drop = set(['source', 'original_shape'])
@@ -154,9 +154,8 @@ def _extract_nc4_encoding(variable, raise_on_invalid=False, lsd_okay=True,
         valid_encodings.add('least_significant_digit')
 
     if (encoding.get('chunksizes') is not None and
-            (encoding.get('original_shape', variable.shape)
-             != variable.shape) and
-            not raise_on_invalid):
+            (encoding.get('original_shape', variable.shape) !=
+                variable.shape) and not raise_on_invalid):
         del encoding['chunksizes']
 
     for k in safe_to_drop:
@@ -209,6 +208,7 @@ def __init__(self, filename, mode='r', format='NETCDF4', group=None,
         self._opener = opener
         self._filename = filename
         self._mode = 'a' if mode == 'w' else mode
+        self.encoding = {}
         super(NetCDF4DataStore, self).__init__(writer)
 
     def open_store_variable(self, name, var):
@@ -251,6 +251,12 @@ def get_dimensions(self):
         return FrozenOrderedDict((k, len(v))
                                  for k, v in iteritems(self.ds.dimensions))
 
+    def get_encoding(self):
+        encoding = {}
+        encoding['unlimited_dims'] = set(
+            [k for k, v in self.ds.dimensions.items() if v.isunlimited()])
+        return encoding
+
     def set_dimension(self, name, length):
         self.ds.createDimension(name, size=length)
 
@@ -270,16 +276,17 @@ def prepare_variable(self, name, variable, check_encoding=False):
             variable = encode_nc3_variable(variable)
             datatype = variable.dtype
 
-        self.set_necessary_dimensions(variable)
+        unlimited_dims = self.encoding.get('unlimited_dims', set())
+        self.set_necessary_dimensions(variable, unlimited_dims=unlimited_dims)
 
         fill_value = attrs.pop('_FillValue', None)
         if fill_value in ['', '\x00']:
             # these are equivalent to the default FillValue, but netCDF4
             # doesn't like setting fill_value to an empty string
             fill_value = None
 
-        encoding = _extract_nc4_encoding(variable,
-                                         raise_on_invalid=check_encoding)
+        encoding = _extract_nc4_variable_encoding(
+            variable, raise_on_invalid=check_encoding)
         nc4_var = self.ds.createVariable(
             varname=name,
             datatype=datatype,

xarray/backends/pydap_.py

@@ -62,6 +62,7 @@ class PydapDataStore(AbstractDataStore):
     def __init__(self, url):
         import pydap.client
         self.ds = pydap.client.open_url(url)
+        self.encoding = {}
 
     def open_store_variable(self, var):
         data = indexing.LazilyIndexedArray(PydapArrayWrapper(var))

xarray/backends/pynio_.py

@@ -42,6 +42,7 @@ def __init__(self, filename, mode='r'):
         self.ds = opener()
         self._opener = opener
         self._mode = mode
+        self.encoding = {}
 
     def open_store_variable(self, name, var):
         data = indexing.LazilyIndexedArray(NioArrayWrapper(name, self))
@@ -57,5 +58,11 @@ def get_attrs(self):
     def get_dimensions(self):
         return Frozen(self.ds.dimensions)
 
+    def get_encoding(self):
+        encoding = {}
+        encoding['unlimited_dims'] = set(
+            [k for k in self.ds.dimensions if self.ds.unlimited(k)])
+        return encoding
+
     def close(self):
         self.ds.close()

xarray/backends/scipy_.py

@@ -8,7 +8,7 @@
 import warnings
 
 from .. import Variable
-from ..core.pycompat import iteritems, basestring, OrderedDict
+from ..core.pycompat import iteritems, OrderedDict
 from ..core.utils import Frozen, FrozenOrderedDict
 from ..core.indexing import NumpyIndexingAdapter
 
@@ -102,6 +102,7 @@ def __init__(self, filename_or_obj, mode='r', format=None, group=None,
         self.ds = opener()
         self._opener = opener
         self._mode = mode
+        self.encoding = {}
 
         super(ScipyDataStore, self).__init__(writer)
 
@@ -116,9 +117,19 @@ def get_variables(self):
     def get_attrs(self):
         return Frozen(_decode_attrs(self.ds._attributes))
 
+    def _get_unlimited_dimensions(self):
+        return set(k for k, v in iteritems(self.ds.dimensions) if v is None)
+
     def get_dimensions(self):
+        self._unlimited_dimensions = self._get_unlimited_dimensions()
         return Frozen(self.ds.dimensions)
 
+    def get_encoding(self):
+        encoding = {}
+        encoding['unlimited_dims'] = set(
+            [k for k, v in self.ds.dimensions.items() if v is None])
+        return encoding
+
     def set_dimension(self, name, length):
         if name in self.dimensions:
             raise ValueError('%s does not support modifying dimensions'
@@ -140,7 +151,12 @@ def prepare_variable(self, name, variable, check_encoding=False):
             raise ValueError('unexpected encoding for scipy backend: %r'
                              % list(variable.encoding))
 
-        self.set_necessary_dimensions(variable)
+        unlimited_dims = self.encoding.get('unlimited_dims', set())
+
+        if len(unlimited_dims) > 1:
+            raise ValueError('NETCDF3 only supports one unlimited dimension')
+        self.set_necessary_dimensions(variable, unlimited_dims=unlimited_dims)
+
         data = variable.data
         # nb. this still creates a numpy array in all memory, even though we
         # don't write the data yet; scipy.io.netcdf does not not support

xarray/conventions.py

@@ -950,6 +950,8 @@ def decode_cf(obj, concat_characters=True, mask_and_scale=True,
     ds = Dataset(vars, attrs=attrs)
     ds = ds.set_coords(coord_names.union(extra_coords).intersection(vars))
     ds._file_obj = file_obj
+    ds.encoding = obj.encoding
+
     return ds
 
 

xarray/core/common.py

@@ -4,8 +4,7 @@
 import numpy as np
 import pandas as pd
 
-from .pycompat import (basestring, iteritems, suppress, dask_array_type,
-                       OrderedDict)
+from .pycompat import (basestring, suppress, dask_array_type, OrderedDict)
 from . import formatting
 from .utils import SortedKeysDict, not_implemented, Frozen
 
@@ -751,7 +750,8 @@ def full_like(other, fill_value, dtype=None):
     elif isinstance(other, DataArray):
         return DataArray(
             _full_like_variable(other.variable, fill_value, dtype),
-            dims=other.dims, coords=other.coords, attrs=other.attrs, name=other.name)
+            dims=other.dims, coords=other.coords, attrs=other.attrs,
+            name=other.name)
     elif isinstance(other, Variable):
         return _full_like_variable(other, fill_value, dtype)
     else: