From 55257b86fcfedba1c4be35183e55fa1f2de13667 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 31 Jan 2018 22:04:07 -0800 Subject: [PATCH 001/282] add warning stating that xarray will drop python 2 support at the end of 2018 (#1871) * add warning stating that xarray will drop python 2 support at the end of 2018 * foot note and plan * more details --- doc/installing.rst | 12 +++++++++++- doc/whats-new.rst | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/doc/installing.rst b/doc/installing.rst index e9fd9885b31..b6ec2bd841f 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -6,7 +6,7 @@ Installation Required dependencies --------------------- -- Python 2.7, 3.4, 3.5, or 3.6 +- Python 2.7 [1]_, 3.4, 3.5, or 3.6 - `numpy `__ (1.11 or later) - `pandas `__ (0.18.0 or later) @@ -96,3 +96,13 @@ To run these benchmark tests in a local machine, first install and run ``asv run # this will install some conda environments in ./.asv/envs`` + +.. [1] Xarray plans to drop support for python 2.7 at the end of 2018. This + means that new releases of xarray published after this date will only be + installable on python 3+ environments, but older versions of xarray will + always be available to python 2.7 users. For more information see the + following references: + + - `Xarray Github issue discussing dropping Python 2 `__ + - `Python 3 Statement `__ + - `Tips on porting to Python 3 `__ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index a1fee8d5961..153d2c32959 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -13,6 +13,18 @@ What's New import xarray as xr np.random.seed(123456) +.. warning:: + + Xarray plans to drop support for python 2.7 at the end of 2018. This + means that new releases of xarray published after this date will only be + installable on python 3+ environments, but older versions of xarray will + always be available to python 2.7 users. For more information see the + following references + + - `Xarray Github issue discussing dropping Python 2 `__ + - `Python 3 Statement `__ + - `Tips on porting to Python 3 `__ + .. _whats-new.0.10.1: v0.10.1 (unreleased) From becd77c44d9436a142db4a98de2b30d388ee6d2b Mon Sep 17 00:00:00 2001 From: Jonas Date: Fri, 2 Feb 2018 03:01:46 +0100 Subject: [PATCH 002/282] test decoding num_dates in float types (#1863) --- xarray/tests/test_coding_times.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index f8ac3d3b58b..eb5c03b1f95 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -27,6 +27,8 @@ def test_cf_datetime(self): import netCDF4 as nc4 for num_dates, units in [ (np.arange(10), 'days since 2000-01-01'), + (np.arange(10).astype('float64'), 'days since 2000-01-01'), + (np.arange(10).astype('float32'), 'days since 2000-01-01'), (np.arange(10).reshape(2, 5), 'days since 2000-01-01'), (12300 + np.arange(5), 'hours since 1680-01-01 00:00:00'), # here we add a couple minor formatting errors to test From e9b98d032adf75509c430056c086389f5f3134fc Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Fri, 2 Feb 2018 20:10:53 +0100 Subject: [PATCH 003/282] Fix rasterio example in docs (#1881) * Fix rasterio example in docs * Review --- doc/conf.py | 10 +++++++- doc/gallery/plot_rasterio.py | 9 ++++--- doc/gallery/plot_rasterio_rgb.py | 43 ++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 doc/gallery/plot_rasterio_rgb.py diff --git a/doc/conf.py b/doc/conf.py index eb71c926375..f92946ccc05 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -20,6 +20,8 @@ import datetime import importlib +allowed_failures = [] + print("python exec:", sys.executable) print("sys.path:", sys.path) for name in ('numpy scipy pandas matplotlib dask IPython seaborn ' @@ -32,6 +34,11 @@ print("%s: %s, %s" % (name, module.__version__, fname)) except ImportError: print("no %s" % name) + if name == 'rasterio': + # not having rasterio should not break the build process + allowed_failures = ['gallery/plot_rasterio_rgb.py', + 'gallery/plot_rasterio.py' + ] import xarray print("xarray: %s, %s" % (xarray.__version__, xarray.__file__)) @@ -62,7 +69,8 @@ sphinx_gallery_conf = {'examples_dirs': 'gallery', 'gallery_dirs': 'auto_gallery', - 'backreferences_dir': False + 'backreferences_dir': False, + 'expected_failing_examples': allowed_failures } autosummary_generate = True diff --git a/doc/gallery/plot_rasterio.py b/doc/gallery/plot_rasterio.py index 2ec58b884eb..d5234950702 100644 --- a/doc/gallery/plot_rasterio.py +++ b/doc/gallery/plot_rasterio.py @@ -13,7 +13,7 @@ These new coordinates might be handy for plotting and indexing, but it should be kept in mind that a grid which is regular in projection coordinates will likely be irregular in lon/lat. It is often recommended to work in the data's -original map projection. +original map projection (see :ref:`recipes.rasterio_rgb`). """ import os @@ -44,10 +44,13 @@ da.coords['lon'] = (('y', 'x'), lon) da.coords['lat'] = (('y', 'x'), lat) +# Compute a greyscale out of the rgb image +greyscale = da.mean(dim='band') + # Plot on a map ax = plt.subplot(projection=ccrs.PlateCarree()) -da.plot.imshow(ax=ax, x='lon', y='lat', rgb='band', - transform=ccrs.PlateCarree()) +greyscale.plot(ax=ax, x='lon', y='lat', transform=ccrs.PlateCarree(), + cmap='Greys_r', add_colorbar=False) ax.coastlines('10m', color='r') plt.show() diff --git a/doc/gallery/plot_rasterio_rgb.py b/doc/gallery/plot_rasterio_rgb.py new file mode 100644 index 00000000000..35c1d0448fe --- /dev/null +++ b/doc/gallery/plot_rasterio_rgb.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +.. _recipes.rasterio_rgb: + +============================ +imshow() and map projections +============================ + +Using rasterio's projection information for more accurate plots. + +This example extends :ref:`recipes.rasterio` and plots the image in the +original map projection instead of relying on pcolormesh and a map +transformation. +""" + +import os +import urllib.request +import xarray as xr +import cartopy.crs as ccrs +import matplotlib.pyplot as plt + +# Download the file from rasterio's repository +url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif' +urllib.request.urlretrieve(url, 'RGB.byte.tif') + +# Read the data +da = xr.open_rasterio('RGB.byte.tif') + +# Normalize the image +da = da / 255 + +# The data is in UTM projection. We have to set it manually until +# https://github.com/SciTools/cartopy/issues/813 is implemented +crs = ccrs.UTM('18N') + +# Plot on a map +ax = plt.subplot(projection=crs) +da.plot.imshow(ax=ax, rgb='band', transform=crs) +ax.coastlines('10m', color='r') +plt.show() + +# Delete the file +os.remove('RGB.byte.tif') From 23866d0447da2765a0d655138edeefa454d99af1 Mon Sep 17 00:00:00 2001 From: Gabriel Joel Mitchell Date: Sun, 4 Feb 2018 13:43:50 -0800 Subject: [PATCH 004/282] Adds references to online documentation in docstrings of apply_ufunc and open_mfdataset (#1884) --- xarray/backends/api.py | 11 ++++++++--- xarray/core/computation.py | 4 +++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 4359868feae..668fb53899d 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -443,8 +443,8 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, lock=None, data_vars='all', coords='different', **kwargs): """Open multiple files as a single dataset. - Requires dask to be installed. Attributes from the first dataset file - are used for the combined dataset. + Requires dask to be installed. See documentation for details on dask [1]. + Attributes from the first dataset file are used for the combined dataset. Parameters ---------- @@ -458,7 +458,7 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, If int, chunk each dimension by ``chunks``. By default, chunks will be chosen to load entire input files into memory at once. This has a major impact on performance: please see the - full documentation for more details. + full documentation for more details [2]. concat_dim : None, str, DataArray or Index, optional Dimension to concatenate files along. This argument is passed on to :py:func:`xarray.auto_combine` along with the dataset objects. You only @@ -533,6 +533,11 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, -------- auto_combine open_dataset + + References + ---------- + .. [1] http://xarray.pydata.org/en/stable/dask.html + .. [2] http://xarray.pydata.org/en/stable/dask.html#chunking-and-performance """ if isinstance(paths, basestring): paths = sorted(glob(paths)) diff --git a/xarray/core/computation.py b/xarray/core/computation.py index f1519027398..5dd9fe78c56 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -836,7 +836,8 @@ def earth_mover_distance(first_samples, Most of NumPy's builtin functions already broadcast their inputs appropriately for use in `apply`. You may find helper functions such as numpy.broadcast_arrays helpful in writing your function. `apply_ufunc` also - works well with numba's vectorize and guvectorize. + works well with numba's vectorize and guvectorize. Further explanation with + examples are provided in the xarray documentation [3]. See also -------- @@ -848,6 +849,7 @@ def earth_mover_distance(first_samples, ---------- .. [1] http://docs.scipy.org/doc/numpy/reference/ufuncs.html .. [2] http://docs.scipy.org/doc/numpy/reference/c-api.generalized-ufuncs.html + .. [3] http://xarray.pydata.org/en/stable/computation.html#wrapping-custom-computation """ # noqa: E501 # don't error on that URL one line up from .groupby import GroupBy from .dataarray import DataArray From 7357a07806d2493c7cb2765f01d54ec9a8f2c87d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 5 Feb 2018 13:00:01 -0800 Subject: [PATCH 005/282] added contributing guide (#1872) * added contributing guide * remove .github/CONTRIBUTING.md after adding CONTRIBUTING.md to top level of repo * cleanup after review from maxim-lian * remove rebasing section * remove CONTRIBUTING.md * remove gitter * edits based on @shoyer's review * typo --- .github/CONTRIBUTING.md | 42 --- doc/contributing.rst | 809 ++++++++++++++++++++++++++++++++++++++++ doc/index.rst | 3 +- doc/whats-new.rst | 4 +- 4 files changed, 814 insertions(+), 44 deletions(-) delete mode 100644 .github/CONTRIBUTING.md create mode 100644 doc/contributing.rst diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index ce8c4d00c3f..00000000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1,42 +0,0 @@ -# Contributing to xarray - -## Usage questions - -The best places to submit questions about how to use xarray are -[Stack Overflow](https://stackoverflow.com/questions/tagged/python-xarray) and -the [xarray Google group](https://groups.google.com/forum/#!forum/xarray). - -## Reporting issues - -When reporting issues please include as much detail as possible about your -operating system, xarray version and python version. Whenever possible, please -also include a brief, self-contained code example that demonstrates the problem. - -## Contributing code - -Thanks for your interest in contributing code to xarray! - -- If you are new to Git or Github, please take a minute to read through a few tutorials - on [Git](https://git-scm.com/docs/gittutorial) and [GitHub](https://guides.github.com/). -- The basic workflow for contributing to xarray is: - 1. [Fork](https://help.github.com/articles/fork-a-repo/) the xarray repository - 2. [Clone](https://help.github.com/articles/cloning-a-repository/) the xarray repository to create a local copy on your computer: - ``` - git clone git@github.com:${user}/xarray.git - cd xarray - ``` - 3. Create a branch for your changes - ``` - git checkout -b name-of-your-branch - ``` - 4. Make change to your local copy of the xarray repository - 5. Commit those changes - ``` - git add file1 file2 file3 - git commit -m 'a descriptive commit message' - ``` - 6. Push your updated branch to your fork - ``` - git push origin name-of-your-branch - ``` - 7. [Open a pull request](https://help.github.com/articles/creating-a-pull-request/) to the pydata/xarray repository. diff --git a/doc/contributing.rst b/doc/contributing.rst new file mode 100644 index 00000000000..2220ab8b1a0 --- /dev/null +++ b/doc/contributing.rst @@ -0,0 +1,809 @@ +.. _contributing: + +********************** +Contributing to xarray +********************** + +.. contents:: Table of contents: + :local: + +.. note:: + + Large parts of this document came from the `Pandas Contributing + Guide `_. + +Where to start? +=============== + +All contributions, bug reports, bug fixes, documentation improvements, +enhancements, and ideas are welcome. + +If you are brand new to xarray or open-source development, we recommend going +through the `GitHub "issues" tab `_ +to find issues that interest you. There are a number of issues listed under +`Documentation `_ +and `good first issue +`_ +where you could start out. Once you've found an interesting issue, you can +return here to get your development environment setup. + +Feel free to ask questions on the `mailing list +`_. + +.. _contributing.bug_reports: + +Bug reports and enhancement requests +==================================== + +Bug reports are an important part of making *xarray* more stable. Having a complete bug report +will allow others to reproduce the bug and provide insight into fixing. See +`this stackoverflow article `_ for tips on +writing a good bug report. + +Trying the bug-producing code out on the *master* branch is often a worthwhile exercise +to confirm the bug still exists. It is also worth searching existing bug reports and pull requests +to see if the issue has already been reported and/or fixed. + +Bug reports must: + +#. Include a short, self-contained Python snippet reproducing the problem. + You can format the code nicely by using `GitHub Flavored Markdown + `_:: + + ```python + >>> from xarray import Dataset + >>> df = Dataset(...) + ... + ``` + +#. Include the full version string of *xarray* and its dependencies. You can use the built in function:: + + >>> import xarray as xr + >>> xr.show_versions() + +#. Explain why the current behavior is wrong/not desired and what you expect instead. + +The issue will then show up to the *xarray* community and be open to comments/ideas from others. + +.. _contributing.github: + +Working with the code +===================== + +Now that you have an issue you want to fix, enhancement to add, or documentation to improve, +you need to learn how to work with GitHub and the *xarray* code base. + +.. _contributing.version_control: + +Version control, Git, and GitHub +-------------------------------- + +To the new user, working with Git is one of the more daunting aspects of contributing to *xarray*. +It can very quickly become overwhelming, but sticking to the guidelines below will help keep the process +straightforward and mostly trouble free. As always, if you are having difficulties please +feel free to ask for help. + +The code is hosted on `GitHub `_. To +contribute you will need to sign up for a `free GitHub account +`_. We use `Git `_ for +version control to allow many people to work together on the project. + +Some great resources for learning Git: + +* the `GitHub help pages `_. +* the `NumPy's documentation `_. +* Matthew Brett's `Pydagogue `_. + +Getting started with Git +------------------------ + +`GitHub has instructions `__ for installing git, +setting up your SSH key, and configuring git. All these steps need to be completed before +you can work seamlessly between your local repository and GitHub. + +.. _contributing.forking: + +Forking +------- + +You will need your own fork to work on the code. Go to the `xarray project +page `_ and hit the ``Fork`` button. You will +want to clone your fork to your machine:: + + git clone https://github.com/your-user-name/xarray.git + cd xarray + git remote add upstream https://github.com/pydata/xarray.git + +This creates the directory `xarray` and connects your repository to +the upstream (main project) *xarray* repository. + +.. _contributing.dev_env: + +Creating a development environment +---------------------------------- + +To test out code changes, you'll need to build xarray from source, which +requires a Python environment. If you're making documentation changes, you can +skip to :ref:`contributing.documentation` but you won't be able to build the +documentation locally before pushing your changes. + +.. _contributiong.dev_python: + +Creating a Python Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before starting any development, you'll need to create an isolated xarray +development environment: + +- Install either `Anaconda `_ or `miniconda + `_ +- Make sure your conda is up to date (``conda update conda``) +- Make sure that you have :ref:`cloned the repository ` +- ``cd`` to the *xarray* source directory + +We'll now kick off a two-step process: + +1. Install the build dependencies +2. Build and install xarray + +.. code-block:: none + + # Create and activate the build environment + conda env create -f ci/requirements-py36.yml + conda activate test_env + + # or with older versions of Anaconda: + source activate test_env + + # Build and install xarray + python setup.py develop + +At this point you should be able to import xarray from your locally built version:: + + $ python # start an interpreter + >>> import xarray + >>> xarray.__version__ + '0.10.0+dev46.g015daca' + +This will create the new environment, and not touch any of your existing environments, +nor any existing Python installation. + +To view your environments:: + + conda info -e + +To return to your root environment:: + + conda deactivate + +See the full conda docs `here `__. + +Creating a branch +----------------- + +You want your master branch to reflect only production-ready code, so create a +feature branch for making your changes. For example:: + + git branch shiny-new-feature + git checkout shiny-new-feature + +The above can be simplified to:: + + git checkout -b shiny-new-feature + +This changes your working directory to the shiny-new-feature branch. Keep any +changes in this branch specific to one bug or feature so it is clear +what the branch brings to *xarray*. You can have many "shiny-new-features" +and switch in between them using the ``git checkout`` command. + +To update this branch, you need to retrieve the changes from the master branch:: + + git fetch upstream + git rebase upstream/master + +This will replay your commits on top of the latest xarray git master. If this +leads to merge conflicts, you must resolve these before submitting your pull +request. If you have uncommitted changes, you will need to ``stash`` them prior +to updating. This will effectively store your changes and they can be reapplied +after updating. + +.. _contributing.documentation: + +Contributing to the documentation +================================= + +If you're not the developer type, contributing to the documentation is still of +huge value. You don't even have to be an expert on *xarray* to do so! In fact, +there are sections of the docs that are worse off after being written by +experts. If something in the docs doesn't make sense to you, updating the +relevant section after you figure it out is a great way to ensure it will help +the next person. + +.. contents:: Documentation: + :local: + + +About the *xarray* documentation +-------------------------------- + +The documentation is written in **reStructuredText**, which is almost like writing +in plain English, and built using `Sphinx `__. The +Sphinx Documentation has an excellent `introduction to reST +`__. Review the Sphinx docs to perform more +complex changes to the documentation as well. + +Some other important things to know about the docs: + +- The *xarray* documentation consists of two parts: the docstrings in the code + itself and the docs in this folder ``xarray/doc/``. + + The docstrings are meant to provide a clear explanation of the usage of the + individual functions, while the documentation in this folder consists of + tutorial-like overviews per topic together with some other information + (what's new, installation, etc). + +- The docstrings follow the **Numpy Docstring Standard**, which is used widely + in the Scientific Python community. This standard specifies the format of + the different sections of the docstring. See `this document + `_ + for a detailed explanation, or look at some of the existing functions to + extend it in a similar manner. + +- The tutorials make heavy use of the `ipython directive + `_ sphinx extension. + This directive lets you put code in the documentation which will be run + during the doc build. For example:: + + .. ipython:: python + + x = 2 + x**3 + + will be rendered as:: + + In [1]: x = 2 + + In [2]: x**3 + Out[2]: 8 + + Almost all code examples in the docs are run (and the output saved) during the + doc build. This approach means that code examples will always be up to date, + but it does make the doc building a bit more complex. + +- Our API documentation in ``doc/api.rst`` houses the auto-generated + documentation from the docstrings. For classes, there are a few subtleties + around controlling which methods and attributes have pages auto-generated. + + Every method should be included in a ``toctree`` in ``api.rst``, else Sphinx + will emit a warning. + + +How to build the *xarray* documentation +--------------------------------------- + +Requirements +~~~~~~~~~~~~ + +First, you need to have a development environment to be able to build xarray +(see the docs on :ref:`creating a development environment above `). + +Building the documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +So how do you build the docs? Navigate to your local +``xarray/doc/`` directory in the console and run:: + + make html + +Then you can find the HTML output in the folder ``xarray/doc/build/html/``. + +The first time you build the docs, it will take quite a while because it has to run +all the code examples and build all the generated docstring pages. In subsequent +evocations, sphinx will try to only build the pages that have been modified. + +If you want to do a full clean build, do:: + + make clean + make html + +.. _contributing.code: + +Contributing to the code base +============================= + +.. contents:: Code Base: + :local: + +Code standards +-------------- + +Writing good code is not just about what you write. It is also about *how* you +write it. During :ref:`Continuous Integration ` testing, several +tools will be run to check your code for stylistic errors. +Generating any warnings will cause the test to fail. +Thus, good style is a requirement for submitting code to *xarray*. + +In addition, because a lot of people use our library, it is important that we +do not make sudden changes to the code that could have the potential to break +a lot of user code as a result, that is, we need it to be as *backwards compatible* +as possible to avoid mass breakages. + +Python (PEP8) +~~~~~~~~~~~~~ + +*xarray* uses the `PEP8 `_ standard. +There are several tools to ensure you abide by this standard. Here are *some* of +the more common ``PEP8`` issues: + + - we restrict line-length to 79 characters to promote readability + - passing arguments should have spaces after commas, e.g. ``foo(arg1, arg2, kw1='bar')`` + +:ref:`Continuous Integration ` will run +the `flake8 `_ tool +and report any stylistic errors in your code. Therefore, it is helpful before +submitting code to run the check yourself on the diff:: + + git diff master -u -- "*.py" | flake8 --diff + +This command will catch any stylistic errors in your changes specifically, but +be beware it may not catch all of them. For example, if you delete the only +usage of an imported function, it is stylistically incorrect to import an +unused function. However, style-checking the diff will not catch this because +the actual import is not part of the diff. Thus, for completeness, you should +run this command, though it will take longer:: + + flake8 xarray + +Backwards Compatibility +~~~~~~~~~~~~~~~~~~~~~~~ + +Please try to maintain backward compatibility. *xarray* has growing number of users with +lots of existing code, so don't break it if at all possible. If you think breakage is, +required clearly state why as part of the pull request. Also, be careful when changing +method signatures and add deprecation warnings where needed. Also, add the deprecated +sphinx directive to the deprecated functions or methods. + +.. _contributing.ci: + +Testing With Continuous Integration +----------------------------------- + +The *xarray* test suite will run automatically on `Travis-CI `__, +and `Appveyor `__, continuous integration services, once +your pull request is submitted. However, if you wish to run the test suite on a +branch prior to submitting the pull request, then the continuous integration +services need to be hooked to your GitHub repository. Instructions are here +for `Travis-CI `__, and +`Appveyor `__. + +A pull-request will be considered for merging when you have an all 'green' build. If any tests are failing, +then you will get a red 'X', where you can click through to see the individual failed tests. +This is an example of a green build. + +.. image:: _static/ci.png + +.. note:: + + Each time you push to *your* fork, a *new* run of the tests will be triggered on the CI. Appveyor will auto-cancel + any non-currently-running tests for that same pull-request. You can also enable the auto-cancel feature for + `Travis-CI here `__. + +.. _contributing.tdd: + + +Test-driven development/code writing +------------------------------------ + +*xarray* is serious about testing and strongly encourages contributors to embrace +`test-driven development (TDD) `_. +This development process "relies on the repetition of a very short development cycle: +first the developer writes an (initially failing) automated test case that defines a desired +improvement or new function, then produces the minimum amount of code to pass that test." +So, before actually writing any code, you should write your tests. Often the test can be +taken from the original GitHub issue. However, it is always worth considering additional +use cases and writing corresponding tests. + +Adding tests is one of the most common requests after code is pushed to *xarray*. Therefore, +it is worth getting in the habit of writing tests ahead of time so this is never an issue. + +Like many packages, *xarray* uses `pytest +`_ and the convenient +extensions in `numpy.testing +`_. + +Writing tests +~~~~~~~~~~~~~ + +All tests should go into the ``tests`` subdirectory of the specific package. +This folder contains many current examples of tests, and we suggest looking to these for +inspiration. If your test requires working with files or +network connectivity, there is more information on the `testing page +`_ of the wiki. + +The ``xarray.testing`` module has many special ``assert`` functions that +make it easier to make statements about whether DataArray or Dataset objects are +equivalent. The easiest way to verify that your code is correct is to +explicitly construct the result you expect, then compare the actual result to +the expected correct result:: + + def test_constructor_from_0d(self): + expected = Dataset({None: ([], 0)})[None] + actual = DataArray(0) + assert_identical(expected, actual) + +Transitioning to ``pytest`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*xarray* existing test structure is *mostly* classed based, meaning that you will typically find tests wrapped in a class. + +.. code-block:: python + + class TestReallyCoolFeature(object): + .... + +Going forward, we are moving to a more *functional* style using the +`pytest `__ framework, which offers a richer +testing framework that will facilitate testing and developing. Thus, instead of +writing test classes, we will write test functions like this: + +.. code-block:: python + + def test_really_cool_feature(): + .... + +Using ``pytest`` +~~~~~~~~~~~~~~~~ + +Here is an example of a self-contained set of tests that illustrate multiple +features that we like to use. + +- functional style: tests are like ``test_*`` and *only* take arguments that are either fixtures or parameters +- ``pytest.mark`` can be used to set metadata on test functions, e.g. ``skip`` or ``xfail``. +- using ``parametrize``: allow testing of multiple cases +- to set a mark on a parameter, ``pytest.param(..., marks=...)`` syntax should be used +- ``fixture``, code for object construction, on a per-test basis +- using bare ``assert`` for scalars and truth-testing +- ``tm.assert_series_equal`` (and its counter part ``tm.assert_frame_equal``), for xarray object comparisons. +- the typical pattern of constructing an ``expected`` and comparing versus the ``result`` + +We would name this file ``test_cool_feature.py`` and put in an appropriate place in the ``xarray/tests/`` structure. + +.. TODO: confirm that this actually works + +.. code-block:: python + + import pytest + import numpy as np + import xarray as xr + from xarray.testing import assert_equal + + + @pytest.mark.parametrize('dtype', ['int8', 'int16', 'int32', 'int64']) + def test_dtypes(dtype): + assert str(np.dtype(dtype)) == dtype + + + @pytest.mark.parametrize('dtype', ['float32', + pytest.param('int16', marks=pytest.mark.skip), + pytest.param('int32', marks=pytest.mark.xfail( + reason='to show how it works'))]) + def test_mark(dtype): + assert str(np.dtype(dtype)) == 'float32' + + + @pytest.fixture + def dataarray(): + return xr.DataArray([1, 2, 3]) + + + @pytest.fixture(params=['int8', 'int16', 'int32', 'int64']) + def dtype(request): + return request.param + + + def test_series(dataarray, dtype): + result = dataarray.astype(dtype) + assert result.dtype == dtype + + expected = xr.DataArray(np.array([1, 2, 3], dtype=dtype)) + assert_equal(result, expected) + + + +A test run of this yields + +.. code-block:: shell + + ((xarray) $ pytest test_cool_feature.py -v + =============================== test session starts ================================ + platform darwin -- Python 3.6.4, pytest-3.2.1, py-1.4.34, pluggy-0.4.0 -- + cachedir: ../../.cache + plugins: cov-2.5.1, hypothesis-3.23.0 + collected 11 items + + test_cool_feature.py::test_dtypes[int8] PASSED + test_cool_feature.py::test_dtypes[int16] PASSED + test_cool_feature.py::test_dtypes[int32] PASSED + test_cool_feature.py::test_dtypes[int64] PASSED + test_cool_feature.py::test_mark[float32] PASSED + test_cool_feature.py::test_mark[int16] SKIPPED + test_cool_feature.py::test_mark[int32] xfail + test_cool_feature.py::test_series[int8] PASSED + test_cool_feature.py::test_series[int16] PASSED + test_cool_feature.py::test_series[int32] PASSED + test_cool_feature.py::test_series[int64] PASSED + + ================== 9 passed, 1 skipped, 1 xfailed in 1.83 seconds ================== + +Tests that we have ``parametrized`` are now accessible via the test name, for +example we could run these with ``-k int8`` to sub-select *only* those tests +which match ``int8``. + + +.. code-block:: shell + + ((xarray) bash-3.2$ pytest test_cool_feature.py -v -k int8 + =========================== test session starts =========================== + platform darwin -- Python 3.6.2, pytest-3.2.1, py-1.4.31, pluggy-0.4.0 + collected 11 items + + test_cool_feature.py::test_dtypes[int8] PASSED + test_cool_feature.py::test_series[int8] PASSED + + +Running the test suite +---------------------- + +The tests can then be run directly inside your Git clone (without having to +install *xarray*) by typing:: + + pytest xarray + +The tests suite is exhaustive and takes a few minutes. Often it is +worth running only a subset of tests first around your changes before running the +entire suite. + +The easiest way to do this is with:: + + pytest xarray/path/to/test.py -k regex_matching_test_name + +Or with one of the following constructs:: + + pytest xarray/tests/[test-module].py + pytest xarray/tests/[test-module].py::[TestClass] + pytest xarray/tests/[test-module].py::[TestClass]::[test_method] + +Using `pytest-xdist `_, one can +speed up local testing on multicore machines. To use this feature, you will +need to install `pytest-xdist` via:: + + pip install pytest-xdist + + +Then, run pytest with the optional -n argument: + + pytest xarray -n 4 + +This can significantly reduce the time it takes to locally run tests before +submitting a pull request. + +For more, see the `pytest `_ documentation. + +Running the performance test suite +---------------------------------- + +Performance matters and it is worth considering whether your code has introduced +performance regressions. *xarray* is starting to write a suite of benchmarking tests +using `asv `__ +to enable easy monitoring of the performance of critical *xarray* operations. +These benchmarks are all found in the ``xarray/asv_bench`` directory. asv +supports both python2 and python3. + +To use all features of asv, you will need either ``conda`` or +``virtualenv``. For more details please check the `asv installation +webpage `_. + +To install asv:: + + pip install git+https://github.com/spacetelescope/asv + +If you need to run a benchmark, change your directory to ``asv_bench/`` and run:: + + asv continuous -f 1.1 upstream/master HEAD + +You can replace ``HEAD`` with the name of the branch you are working on, +and report benchmarks that changed by more than 10%. +The command uses ``conda`` by default for creating the benchmark +environments. If you want to use virtualenv instead, write:: + + asv continuous -f 1.1 -E virtualenv upstream/master HEAD + +The ``-E virtualenv`` option should be added to all ``asv`` commands +that run benchmarks. The default value is defined in ``asv.conf.json``. + +Running the full benchmark suite can take up to one hour and use up a few GBs of RAM. +Usually it is sufficient to paste only a subset of the results into the pull +request to show that the committed changes do not cause unexpected performance +regressions. You can run specific benchmarks using the ``-b`` flag, which +takes a regular expression. For example, this will only run tests from a +``xarray/asv_bench/benchmarks/groupby.py`` file:: + + asv continuous -f 1.1 upstream/master HEAD -b ^groupby + +If you want to only run a specific group of tests from a file, you can do it +using ``.`` as a separator. For example:: + + asv continuous -f 1.1 upstream/master HEAD -b groupby.GroupByMethods + +will only run the ``GroupByMethods`` benchmark defined in ``groupby.py``. + +You can also run the benchmark suite using the version of ``xarray`` +already installed in your current Python environment. This can be +useful if you do not have virtualenv or conda, or are using the +``setup.py develop`` approach discussed above; for the in-place build +you need to set ``PYTHONPATH``, e.g. +``PYTHONPATH="$PWD/.." asv [remaining arguments]``. +You can run benchmarks using an existing Python +environment by:: + + asv run -e -E existing + +or, to use a specific Python interpreter,:: + + asv run -e -E existing:python3.5 + +This will display stderr from the benchmarks, and use your local +``python`` that comes from your ``$PATH``. + +Information on how to write a benchmark and how to use asv can be found in the +`asv documentation `_. + +The ``xarray`` benchmarking suite is run remotely and the results are +available `here `. + +Contributing your changes to *xarray* +===================================== + +Committing your code +-------------------- + +Keep style fixes to a separate commit to make your pull request more readable. + +Once you've made changes, you can see them by typing:: + + git status + +If you have created a new file, it is not being tracked by git. Add it by typing:: + + git add path/to/file-to-be-added.py + +Doing 'git status' again should give something like:: + + # On branch shiny-new-feature + # + # modified: /relative/path/to/file-you-added.py + # + +Finally, commit your changes to your local repository with an explanatory message. +*Xarray* uses a convention for commit message prefixes and layout. Here are +some common prefixes along with general guidelines for when to use them: + + * ENH: Enhancement, new functionality + * BUG: Bug fix + * DOC: Additions/updates to documentation + * TST: Additions/updates to tests + * BLD: Updates to the build process/scripts + * PERF: Performance improvement + * CLN: Code cleanup + +The following defines how a commit message should be structured. Please reference the +relevant GitHub issues in your commit message using GH1234 or #1234. Either style +is fine, but the former is generally preferred: + + * a subject line with `< 80` chars. + * One blank line. + * Optionally, a commit message body. + +Now you can commit your changes in your local repository:: + + git commit -m + +Pushing your changes +-------------------- + +When you want your changes to appear publicly on your GitHub page, push your +forked feature branch's commits:: + + git push origin shiny-new-feature + +Here ``origin`` is the default name given to your remote repository on GitHub. +You can see the remote repositories:: + + git remote -v + +If you added the upstream repository as described above you will see something +like:: + + origin git@github.com:yourname/xarray.git (fetch) + origin git@github.com:yourname/xarray.git (push) + upstream git://github.com/pydata/xarray.git (fetch) + upstream git://github.com/pydata/xarray.git (push) + +Now your code is on GitHub, but it is not yet a part of the *xarray* project. For that to +happen, a pull request needs to be submitted on GitHub. + +Review your code +---------------- + +When you're ready to ask for a code review, file a pull request. Before you do, once +again make sure that you have followed all the guidelines outlined in this document +regarding code style, tests, performance tests, and documentation. You should also +double check your branch changes against the branch it was based on: + +#. Navigate to your repository on GitHub -- https://github.com/your-user-name/xarray +#. Click on ``Branches`` +#. Click on the ``Compare`` button for your feature branch +#. Select the ``base`` and ``compare`` branches, if necessary. This will be ``master`` and + ``shiny-new-feature``, respectively. + +Finally, make the pull request +------------------------------ + +If everything looks good, you are ready to make a pull request. A pull request is how +code from a local repository becomes available to the GitHub community and can be looked +at and eventually merged into the master version. This pull request and its associated +changes will eventually be committed to the master branch and available in the next +release. To submit a pull request: + +#. Navigate to your repository on GitHub +#. Click on the ``Pull Request`` button +#. You can then click on ``Commits`` and ``Files Changed`` to make sure everything looks + okay one last time +#. Write a description of your changes in the ``Preview Discussion`` tab +#. Click ``Send Pull Request``. + +This request then goes to the repository maintainers, and they will review +the code. If you need to make more changes, you can make them in +your branch, add them to a new commit, push them to GitHub, and the pull request +will be automatically updated. Pushing them to GitHub again is done by:: + + git push origin shiny-new-feature + +This will automatically update your pull request with the latest code and restart the +:ref:`Continuous Integration ` tests. + + +Delete your merged branch (optional) +------------------------------------ + +Once your feature branch is accepted into upstream, you'll probably want to get rid of +the branch. First, merge upstream master into your branch so git knows it is safe to +delete your branch:: + + git fetch upstream + git checkout master + git merge upstream/master + +Then you can do:: + + git branch -d shiny-new-feature + +Make sure you use a lower-case ``-d``, or else git won't warn you if your feature +branch has not actually been merged. + +The branch will still exist on GitHub, so to delete it there do:: + + git push origin --delete shiny-new-feature diff --git a/doc/index.rst b/doc/index.rst index 607dce2ed50..8389d598bcf 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -35,7 +35,6 @@ Documentation .. toctree:: :maxdepth: 1 - whats-new why-xarray faq examples @@ -53,6 +52,8 @@ Documentation plotting api internals + contributing + whats-new See also -------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 153d2c32959..ecd9877c496 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -34,10 +34,12 @@ Documentation ~~~~~~~~~~~~~ - Added apply_ufunc example to toy weather data page (:issue:`1844`). - By `Liam Brannigan ` _. + By `Liam Brannigan `_. - New entry `Why don’t aggregations return Python scalars?` in the :doc:`faq` (:issue:`1726`). By `0x0L `_. +- Added a new contributors guide (:issue:`640`) + By `Joe Hamman `_. Enhancements ~~~~~~~~~~~~ From 518436576b1661bb04e95987abf7678749b3d634 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Tue, 6 Feb 2018 20:59:03 +0100 Subject: [PATCH 006/282] Use pip install -e in contributing docs (#1891) --- doc/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 2220ab8b1a0..c389321764f 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -156,7 +156,7 @@ We'll now kick off a two-step process: source activate test_env # Build and install xarray - python setup.py develop + pip install -e . At this point you should be able to import xarray from your locally built version:: From 1d3239982db9778e89a48fe55b01d0a525673a7a Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Wed, 7 Feb 2018 09:40:33 +0100 Subject: [PATCH 007/282] Simplify some rasterio tests (#1890) * Simplify some rasterio tests * pep8 --- doc/whats-new.rst | 6 ++++++ xarray/tests/test_backends.py | 37 +++++++++++++++-------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index ecd9877c496..f17bb8f0d49 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -129,6 +129,12 @@ Bug fixes - Fix indexing with lists for arrays loaded from netCDF files with ``engine='h5netcdf`` (:issue:`1864`). By `Stephan Hoyer `_. +- Corrected a bug with incorrect coordinates for non-georeferenced geotiff + files (:issue:`1686`). Internally, we now use the rasterio coordinate + transform tool instead of doing the computations ourselves. A + ``parse_coordinates`` kwarg has beed added to :py:func:`~open_rasterio` + (set to ``True`` per default). + By `Fabien Maussion `_. .. _whats-new.0.10.0: diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 73eb49b863b..85b6bdea346 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2178,9 +2178,11 @@ class TestPyNioAutocloseTrue(TestPyNio): @requires_rasterio @contextlib.contextmanager def create_tmp_geotiff(nx=4, ny=3, nz=3, + transform=None, transform_args=[5000, 80000, 1000, 2000.], crs={'units': 'm', 'no_defs': True, 'ellps': 'WGS84', - 'proj': 'utm', 'zone': 18}): + 'proj': 'utm', 'zone': 18}, + open_kwargs={}): # yields a temporary geotiff file and a corresponding expected DataArray import rasterio from rasterio.transform import from_origin @@ -2192,15 +2194,16 @@ def create_tmp_geotiff(nx=4, ny=3, nz=3, else: data_shape = nz, ny, nx write_kwargs = {} - data = np.arange(nz*ny*nx, - dtype=rasterio.float32).reshape(*data_shape) - transform = from_origin(*transform_args) + data = np.arange(nz*ny*nx, dtype=rasterio.float32).reshape(*data_shape) + if transform is None: + transform = from_origin(*transform_args) with rasterio.open( tmp_file, 'w', driver='GTiff', height=ny, width=nx, count=nz, crs=crs, transform=transform, - dtype=rasterio.float32) as s: + dtype=rasterio.float32, + **open_kwargs) as s: s.write(data, **write_kwargs) dx, dy = s.res[0], -s.res[1] @@ -2236,6 +2239,8 @@ def test_utm(self): assert isinstance(rioda.attrs['res'], tuple) assert isinstance(rioda.attrs['is_tiled'], np.uint8) assert isinstance(rioda.attrs['transform'], tuple) + np.testing.assert_array_equal(rioda.attrs['nodatavals'], + [np.NaN, np.NaN, np.NaN]) # Check no parse coords with xr.open_rasterio(tmp_file, parse_coordinates=False) as rioda: @@ -2243,23 +2248,10 @@ def test_utm(self): assert 'y' not in rioda.coords def test_non_rectilinear(self): - import rasterio from rasterio.transform import from_origin - # Create a geotiff file with 2d coordinates - with create_tmp_file(suffix='.tif') as tmp_file: - # data - nx, ny, nz = 4, 3, 3 - data = np.arange(nx*ny*nz, - dtype=rasterio.float32).reshape(nz, ny, nx) - transform = from_origin(0, 3, 1, 1).rotation(45) - with rasterio.open( - tmp_file, 'w', - driver='GTiff', height=ny, width=nx, count=nz, - transform=transform, - dtype=rasterio.float32) as s: - s.write(data) - + with create_tmp_geotiff(transform=from_origin(0, 3, 1, 1).rotation(45), + crs=None) as (tmp_file, _): # Default is to not parse coords with xr.open_rasterio(tmp_file) as rioda: assert 'x' not in rioda.coords @@ -2278,7 +2270,8 @@ def test_non_rectilinear(self): def test_platecarree(self): with create_tmp_geotiff(8, 10, 1, transform_args=[1, 2, 0.5, 2.], - crs='+proj=latlong') \ + crs='+proj=latlong', + open_kwargs={'nodata': -9765}) \ as (tmp_file, expected): with xr.open_rasterio(tmp_file) as rioda: assert_allclose(rioda, expected) @@ -2286,6 +2279,8 @@ def test_platecarree(self): assert isinstance(rioda.attrs['res'], tuple) assert isinstance(rioda.attrs['is_tiled'], np.uint8) assert isinstance(rioda.attrs['transform'], tuple) + np.testing.assert_array_equal(rioda.attrs['nodatavals'], + [-9765.]) def test_notransform(self): # regression test for https://github.com/pydata/xarray/issues/1686 From cbf4921102e3dbb77b9ca774caa48eebc1b27fc2 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sun, 11 Feb 2018 15:21:18 -0800 Subject: [PATCH 008/282] Support dt.floor(), dt.ceil() and dt.round() accessors. (#1827) * Support dt.floor(), dt.ceil() and dt.round() accessors. * Address comments. * Add dask test + dtype. * Add docstrings. --- doc/time-series.rst | 8 +++ doc/whats-new.rst | 4 +- xarray/core/accessors.py | 92 ++++++++++++++++++++++++++++++++++ xarray/tests/test_accessors.py | 20 ++++++++ 4 files changed, 123 insertions(+), 1 deletion(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index bdf8b1e7f81..4d9a995051a 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -129,6 +129,14 @@ the first letters of the corresponding months. You can use these shortcuts with both Datasets and DataArray coordinates. +In addition, xarray supports rounding operations ``floor``, ``ceil``, and ``round``. These operations require that you supply a `rounding frequency as a string argument.`__ + +__ http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + +.. ipython:: python + + ds['time'].dt.floor('D') + .. _resampling: Resampling and grouped operations diff --git a/doc/whats-new.rst b/doc/whats-new.rst index f17bb8f0d49..115cd9bb54f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -79,6 +79,8 @@ Enhancements (:pull:`1840`), and keeping float16 and float32 as float32 (:issue:`1842`). Correspondingly, encoded variables may also be saved with a smaller dtype. By `Zac Hatfield-Dodds `_. +- `.dt` accessor can now ceil, floor and round timestamps to specified frequency. + By `Deepak Cherian `_. .. _Zarr: http://zarr.readthedocs.io/ @@ -94,7 +96,7 @@ Bug fixes ~~~~~~~~~ - Added warning in api.py of a netCDF4 bug that occurs when the filepath has 88 characters (:issue:`1745`). - By `Liam Brannigan ` _. + By `Liam Brannigan `_. - Fixed encoding of multi-dimensional coordinates in :py:meth:`~Dataset.to_netcdf` (:issue:`1763`). By `Mike Neish `_. diff --git a/xarray/core/accessors.py b/xarray/core/accessors.py index 5052b555c73..b3e7e1ff9a1 100644 --- a/xarray/core/accessors.py +++ b/xarray/core/accessors.py @@ -58,6 +58,43 @@ def _get_date_field(values, name, dtype): return _access_through_series(values, name) +def _round_series(values, name, freq): + """Coerce an array of datetime-like values to a pandas Series and + apply requested rounding + """ + values_as_series = pd.Series(values.ravel()) + method = getattr(values_as_series.dt, name) + field_values = method(freq=freq).values + + return field_values.reshape(values.shape) + + +def _round_field(values, name, freq): + """Indirectly access pandas rounding functions by wrapping data + as a Series and calling through `.dt` attribute. + + Parameters + ---------- + values : np.ndarray or dask.array-like + Array-like container of datetime-like values + name : str (ceil, floor, round) + Name of rounding function + freq : a freq string indicating the rounding resolution + + Returns + ------- + rounded timestamps : same type as values + Array-like of datetime fields accessed for each element in values + + """ + if isinstance(values, dask_array_type): + from dask.array import map_blocks + return map_blocks(_round_series, + values, name, freq=freq, dtype=np.datetime64) + else: + return _round_series(values, name, freq) + + class DatetimeAccessor(object): """Access datetime fields for DataArrays with datetime-like dtypes. @@ -147,3 +184,58 @@ def f(self, dtype=dtype): time = _tslib_field_accessor( "time", "Timestamps corresponding to datetimes", object ) + + def _tslib_round_accessor(self, name, freq): + obj_type = type(self._obj) + result = _round_field(self._obj.data, name, freq) + return obj_type(result, name=name, + coords=self._obj.coords, dims=self._obj.dims) + + def floor(self, freq): + ''' + Round timestamps downward to specified frequency resolution. + + Parameters + ---------- + freq : a freq string indicating the rounding resolution + e.g. 'D' for daily resolution + + Returns + ------- + floor-ed timestamps : same type as values + Array-like of datetime fields accessed for each element in values + ''' + + return self._tslib_round_accessor("floor", freq) + + def ceil(self, freq): + ''' + Round timestamps upward to specified frequency resolution. + + Parameters + ---------- + freq : a freq string indicating the rounding resolution + e.g. 'D' for daily resolution + + Returns + ------- + ceil-ed timestamps : same type as values + Array-like of datetime fields accessed for each element in values + ''' + return self._tslib_round_accessor("ceil", freq) + + def round(self, freq): + ''' + Round timestamps to specified frequency resolution. + + Parameters + ---------- + freq : a freq string indicating the rounding resolution + e.g. 'D' for daily resolution + + Returns + ------- + rounded timestamps : same type as values + Array-like of datetime fields accessed for each element in values + ''' + return self._tslib_round_accessor("round", freq) diff --git a/xarray/tests/test_accessors.py b/xarray/tests/test_accessors.py index 30ea1a88c7a..1fcde8f5a68 100644 --- a/xarray/tests/test_accessors.py +++ b/xarray/tests/test_accessors.py @@ -57,6 +57,9 @@ def test_dask_field_access(self): months = self.times_data.dt.month hours = self.times_data.dt.hour days = self.times_data.dt.day + floor = self.times_data.dt.floor('D') + ceil = self.times_data.dt.ceil('D') + round = self.times_data.dt.round('D') dask_times_arr = da.from_array(self.times_arr, chunks=(5, 5, 50)) dask_times_2d = xr.DataArray(dask_times_arr, @@ -67,6 +70,9 @@ def test_dask_field_access(self): dask_month = dask_times_2d.dt.month dask_day = dask_times_2d.dt.day dask_hour = dask_times_2d.dt.hour + dask_floor = dask_times_2d.dt.floor('D') + dask_ceil = dask_times_2d.dt.ceil('D') + dask_round = dask_times_2d.dt.round('D') # Test that the data isn't eagerly evaluated assert isinstance(dask_year.data, da.Array) @@ -86,6 +92,9 @@ def test_dask_field_access(self): assert_equal(months, dask_month.compute()) assert_equal(days, dask_day.compute()) assert_equal(hours, dask_hour.compute()) + assert_equal(floor, dask_floor.compute()) + assert_equal(ceil, dask_ceil.compute()) + assert_equal(round, dask_round.compute()) def test_seasons(self): dates = pd.date_range(start="2000/01/01", freq="M", periods=12) @@ -95,3 +104,14 @@ def test_seasons(self): seasons = xr.DataArray(seasons) assert_array_equal(seasons.values, dates.dt.season.values) + + def test_rounders(self): + dates = pd.date_range("2014-01-01", "2014-05-01", freq='H') + xdates = xr.DataArray(np.arange(len(dates)), + dims=['time'], coords=[dates]) + assert_array_equal(dates.floor('D').values, + xdates.time.dt.floor('D').values) + assert_array_equal(dates.ceil('D').values, + xdates.time.dt.ceil('D').values) + assert_array_equal(dates.round('D').values, + xdates.time.dt.round('D').values) From ee38ff07e8c48c8542742b3e42dd00dc41eb38f7 Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Mon, 12 Feb 2018 16:08:05 -0500 Subject: [PATCH 009/282] Replace task_state with tasks in dask test (#1904) This internal state was changed in the latest release Fixes #1903 --- xarray/tests/test_distributed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 1d450ff51d4..47bb6cdc2e1 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -81,4 +81,4 @@ def test_async(c, s, a, b): assert not dask.is_dask_collection(w) assert_allclose(x + 10, w) - assert s.task_state + assert s.tasks From 93a4039f6c6eb765f5b2dc1ba286b263a931dac6 Mon Sep 17 00:00:00 2001 From: Zac Hatfield Dodds Date: Tue, 13 Feb 2018 09:12:12 +1100 Subject: [PATCH 010/282] Use correct dtype for RGB image alpha channel (#1893) * Fix alpha channel logic for integer RGB images * Use named argument for concat axis --- doc/gallery/plot_rasterio_rgb.py | 3 --- xarray/plot/plot.py | 8 ++++++-- xarray/tests/test_plot.py | 7 +++++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/gallery/plot_rasterio_rgb.py b/doc/gallery/plot_rasterio_rgb.py index 35c1d0448fe..ec2bbe63218 100644 --- a/doc/gallery/plot_rasterio_rgb.py +++ b/doc/gallery/plot_rasterio_rgb.py @@ -26,9 +26,6 @@ # Read the data da = xr.open_rasterio('RGB.byte.tif') -# Normalize the image -da = da / 255 - # The data is in UTM projection. We have to set it manually until # https://github.com/SciTools/cartopy/issues/813 is implemented crs = ccrs.UTM('18N') diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index d17ceb84e16..97fee5f01c0 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -710,8 +710,12 @@ def imshow(x, y, z, ax, **kwargs): # missing data transparent. We therefore add an alpha channel if # there isn't one, and set it to transparent where data is masked. if z.shape[-1] == 3: - z = np.ma.concatenate((z, np.ma.ones(z.shape[:2] + (1,))), 2) - z = z.copy() + alpha = np.ma.ones(z.shape[:2] + (1,), dtype=z.dtype) + if np.issubdtype(z.dtype, np.integer): + alpha *= 255 + z = np.ma.concatenate((z, alpha), axis=2) + else: + z = z.copy() z[np.any(z.mask, axis=-1), -1] = 0 primitive = ax.imshow(z, **defaults) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 1573577a092..479b1dce9da 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1146,6 +1146,13 @@ def test_normalize_rgb_one_arg_error(self): for kwds in [dict(vmax=-1, vmin=-1.2), dict(vmin=2, vmax=2.1)]: da.plot.imshow(**kwds) + def test_imshow_rgb_values_in_valid_range(self): + da = DataArray(np.arange(75, dtype='uint8').reshape((5, 5, 3))) + _, ax = plt.subplots() + out = da.plot.imshow(ax=ax).get_array() + assert out.dtype == np.uint8 + assert (out[..., :3] == da.values).all() # Compare without added alpha + class TestFacetGrid(PlotTestCase): def setUp(self): From 33660b76057b7dc5b9c7dac77e286bb755b1d68e Mon Sep 17 00:00:00 2001 From: Chris Roth Date: Tue, 13 Feb 2018 12:34:36 -0600 Subject: [PATCH 011/282] Add '_FillValue' to set of valid_encodings for netCDF4 backend (#1869) * Add '_FillValue' to set of valid_encodings for netCDF4 backend * Add additional omit_fill_value tests Add additional tests to prevent future regression of setting _FillValue to None when using the encoding kwarg in to_netcdf. * Fix additional omit_fill_value tests Remove copy/paste line that shouldn't have been in a test. Add additional asserts. Fix indentation. * Fix scipy failure in additional omit_fill_value tests * Added bug-fix documentation for pydata/xarray#1865 --- doc/whats-new.rst | 3 +++ xarray/backends/netCDF4_.py | 2 +- xarray/backends/scipy_.py | 5 +++-- xarray/tests/test_backends.py | 20 ++++++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 115cd9bb54f..875aba512fc 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -128,6 +128,9 @@ Bug fixes - Compatibility fixes to plotting module for Numpy 1.14 and Pandas 0.22 (:issue:`1813`). By `Joe Hamman `_. +- Bug fix in encoding coordinates with ``{'_FillValue': None}`` in netCDF + metadata (:issue:`1865`). + By `Chris Roth `_. - Fix indexing with lists for arrays loaded from netCDF files with ``engine='h5netcdf`` (:issue:`1864`). By `Stephan Hoyer `_. diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index b3cb1d8e49f..3f3364dec56 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -160,7 +160,7 @@ def _extract_nc4_variable_encoding(variable, raise_on_invalid=False, safe_to_drop = set(['source', 'original_shape']) valid_encodings = set(['zlib', 'complevel', 'fletcher32', 'contiguous', - 'chunksizes', 'shuffle']) + 'chunksizes', 'shuffle', '_FillValue']) if lsd_okay: valid_encodings.add('least_significant_digit') diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index dba2e5672a2..a608cff8eb5 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -188,8 +188,9 @@ def encode_variable(self, variable): def prepare_variable(self, name, variable, check_encoding=False, unlimited_dims=None): if check_encoding and variable.encoding: - raise ValueError('unexpected encoding for scipy backend: %r' - % list(variable.encoding)) + if variable.encoding != {'_FillValue': None}: + raise ValueError('unexpected encoding for scipy backend: %r' + % list(variable.encoding)) data = variable.data # nb. this still creates a numpy array in all memory, even though we diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 85b6bdea346..e88fc790571 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -704,6 +704,26 @@ def test_explicitly_omit_fill_value(self): with self.roundtrip(ds) as actual: assert '_FillValue' not in actual.x.encoding + def test_explicitly_omit_fill_value_via_encoding_kwarg(self): + ds = Dataset({'x': ('y', [np.pi, -np.pi])}) + kwargs = dict(encoding={'x': {'_FillValue': None}}) + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + assert '_FillValue' not in actual.x.encoding + self.assertEqual(ds.y.encoding, {}) + + def test_explicitly_omit_fill_value_in_coord(self): + ds = Dataset({'x': ('y', [np.pi, -np.pi])}, coords={'y': [0.0, 1.0]}) + ds.y.encoding['_FillValue'] = None + with self.roundtrip(ds) as actual: + assert '_FillValue' not in actual.y.encoding + + def test_explicitly_omit_fill_value_in_coord_via_encoding_kwarg(self): + ds = Dataset({'x': ('y', [np.pi, -np.pi])}, coords={'y': [0.0, 1.0]}) + kwargs = dict(encoding={'y': {'_FillValue': None}}) + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + assert '_FillValue' not in actual.y.encoding + self.assertEqual(ds.y.encoding, {}) + def test_encoding_same_dtype(self): ds = Dataset({'x': ('y', np.arange(10.0, dtype='f4'))}) kwargs = dict(encoding={'x': {'dtype': 'f4'}}) From 2aa5b8a5c094593569f5bd9ae220d1f2fc0ecda0 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 14 Feb 2018 05:11:47 -0800 Subject: [PATCH 012/282] Use getitem_with_mask in reindex_variables (#1847) * WIP: use getitem_with_mask in reindex_variables * Fix dtype promotion for where * Add whats new * Fix flake8 * Fix test_align_dtype and bool+str promotion * tests and docstring for dtypes.result_type * More dtype promotion fixes, including for concat --- asv_bench/asv.conf.json | 2 +- asv_bench/benchmarks/reindexing.py | 45 +++++++++++++ doc/whats-new.rst | 7 ++ xarray/core/alignment.py | 101 ++++++++++------------------ xarray/core/dtypes.py | 37 ++++++++++ xarray/core/duck_array_ops.py | 32 ++++++++- xarray/core/variable.py | 2 - xarray/tests/test_dataarray.py | 6 ++ xarray/tests/test_dtypes.py | 48 +++++++++++++ xarray/tests/test_duck_array_ops.py | 18 ++++- xarray/tests/test_variable.py | 13 +++- 11 files changed, 234 insertions(+), 77 deletions(-) create mode 100644 asv_bench/benchmarks/reindexing.py create mode 100644 xarray/tests/test_dtypes.py diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json index a2878a7bf50..b5953436387 100644 --- a/asv_bench/asv.conf.json +++ b/asv_bench/asv.conf.json @@ -63,7 +63,7 @@ "netcdf4": [""], "scipy": [""], "bottleneck": ["", null], - "dask": ["", null], + "dask": [""], }, diff --git a/asv_bench/benchmarks/reindexing.py b/asv_bench/benchmarks/reindexing.py new file mode 100644 index 00000000000..0f28eaa4cee --- /dev/null +++ b/asv_bench/benchmarks/reindexing.py @@ -0,0 +1,45 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import xarray as xr + +from . import requires_dask + + +class Reindex(object): + def setup(self): + data = np.random.RandomState(0).randn(1000, 100, 100) + self.ds = xr.Dataset({'temperature': (('time', 'x', 'y'), data)}, + coords={'time': np.arange(1000), + 'x': np.arange(100), + 'y': np.arange(100)}) + + def time_1d_coarse(self): + self.ds.reindex(time=np.arange(0, 1000, 5)).load() + + def time_1d_fine_all_found(self): + self.ds.reindex(time=np.arange(0, 1000, 0.5), method='nearest').load() + + def time_1d_fine_some_missing(self): + self.ds.reindex(time=np.arange(0, 1000, 0.5), method='nearest', + tolerance=0.1).load() + + def time_2d_coarse(self): + self.ds.reindex(x=np.arange(0, 100, 2), y=np.arange(0, 100, 2)).load() + + def time_2d_fine_all_found(self): + self.ds.reindex(x=np.arange(0, 100, 0.5), y=np.arange(0, 100, 0.5), + method='nearest').load() + + def time_2d_fine_some_missing(self): + self.ds.reindex(x=np.arange(0, 100, 0.5), y=np.arange(0, 100, 0.5), + method='nearest', tolerance=0.1).load() + + +class ReindexDask(Reindex): + def setup(self): + requires_dask() + super(ReindexDask, self).setup() + self.ds = self.ds.chunk({'time': 100}) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 875aba512fc..0a3f949b496 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -81,6 +81,9 @@ Enhancements By `Zac Hatfield-Dodds `_. - `.dt` accessor can now ceil, floor and round timestamps to specified frequency. By `Deepak Cherian `_. +- Speed of reindexing/alignment with dask array is orders of magnitude faster + when inserting missing values (:issue:`1847`). + By `Stephan Hoyer `_. .. _Zarr: http://zarr.readthedocs.io/ @@ -140,6 +143,10 @@ Bug fixes ``parse_coordinates`` kwarg has beed added to :py:func:`~open_rasterio` (set to ``True`` per default). By `Fabien Maussion `_. +- Fixed dtype promotion rules in :py:func:`where` and :py:func:`concat` to + match pandas (:issue:`1847`). A combination of strings/numbers or + unicode/bytes now promote to object dtype, instead of strings or unicode. + By `Stephan Hoyer `_. .. _whats-new.0.10.0: diff --git a/xarray/core/alignment.py b/xarray/core/alignment.py index 876245322fa..99dde45b892 100644 --- a/xarray/core/alignment.py +++ b/xarray/core/alignment.py @@ -8,13 +8,11 @@ import numpy as np -from . import duck_array_ops -from . import dtypes from . import utils from .indexing import get_indexer_nd from .pycompat import iteritems, OrderedDict, suppress from .utils import is_full_slice, is_dict_like -from .variable import Variable, IndexVariable +from .variable import IndexVariable def _get_joiner(join): @@ -306,59 +304,51 @@ def reindex_variables(variables, sizes, indexes, indexers, method=None, from .dataarray import DataArray # build up indexers for assignment along each dimension - to_indexers = {} - from_indexers = {} + int_indexers = {} + targets = {} + masked_dims = set() + unchanged_dims = set() + # size of reindexed dimensions new_sizes = {} for name, index in iteritems(indexes): if name in indexers: - target = utils.safe_cast_to_index(indexers[name]) if not index.is_unique: raise ValueError( 'cannot reindex or align along dimension %r because the ' 'index has duplicate values' % name) - indexer = get_indexer_nd(index, target, method, tolerance) + target = utils.safe_cast_to_index(indexers[name]) new_sizes[name] = len(target) - # Note pandas uses negative values from get_indexer_nd to signify - # values that are missing in the index - # The non-negative values thus indicate the non-missing values - to_indexers[name] = indexer >= 0 - if to_indexers[name].all(): - # If an indexer includes no negative values, then the - # assignment can be to a full-slice (which is much faster, - # and means we won't need to fill in any missing values) - to_indexers[name] = slice(None) - - from_indexers[name] = indexer[to_indexers[name]] - if np.array_equal(from_indexers[name], np.arange(len(index))): - # If the indexer is equal to the original index, use a full - # slice object to speed up selection and so we can avoid - # unnecessary copies - from_indexers[name] = slice(None) + + int_indexer = get_indexer_nd(index, target, method, tolerance) + + # We uses negative values from get_indexer_nd to signify + # values that are missing in the index. + if (int_indexer < 0).any(): + masked_dims.add(name) + elif np.array_equal(int_indexer, np.arange(len(index))): + unchanged_dims.add(name) + + int_indexers[name] = int_indexer + targets[name] = target for dim in sizes: if dim not in indexes and dim in indexers: existing_size = sizes[dim] - new_size = utils.safe_cast_to_index(indexers[dim]).size + new_size = indexers[dim].size if existing_size != new_size: raise ValueError( 'cannot reindex or align along dimension %r without an ' 'index because its size %r is different from the size of ' 'the new index %r' % (dim, existing_size, new_size)) - def any_not_full_slices(indexers): - return any(not is_full_slice(idx) for idx in indexers) - - def var_indexers(var, indexers): - return tuple(indexers.get(d, slice(None)) for d in var.dims) - # create variables for the new dataset reindexed = OrderedDict() for dim, indexer in indexers.items(): - if isinstance(indexer, DataArray) and indexer.dims != (dim, ): + if isinstance(indexer, DataArray) and indexer.dims != (dim,): warnings.warn( "Indexer has dimensions {0:s} that are different " "from that to be indexed along {1:s}. " @@ -375,47 +365,24 @@ def var_indexers(var, indexers): for name, var in iteritems(variables): if name not in indexers: - assign_to = var_indexers(var, to_indexers) - assign_from = var_indexers(var, from_indexers) - - if any_not_full_slices(assign_to): - # there are missing values to in-fill - data = var[assign_from].data - dtype, fill_value = dtypes.maybe_promote(var.dtype) - - if isinstance(data, np.ndarray): - shape = tuple(new_sizes.get(dim, size) - for dim, size in zip(var.dims, var.shape)) - new_data = np.empty(shape, dtype=dtype) - new_data[...] = fill_value - # create a new Variable so we can use orthogonal indexing - # use fastpath=True to avoid dtype inference - new_var = Variable(var.dims, new_data, var.attrs, - fastpath=True) - new_var[assign_to] = data - - else: # dask array - data = data.astype(dtype, copy=False) - for axis, indexer in enumerate(assign_to): - if not is_full_slice(indexer): - indices = np.cumsum(indexer)[~indexer] - data = duck_array_ops.insert( - data, indices, fill_value, axis=axis) - new_var = Variable(var.dims, data, var.attrs, - fastpath=True) - - elif any_not_full_slices(assign_from): - # type coercion is not necessary as there are no missing - # values - new_var = var[assign_from] - - else: - # no reindexing is necessary + key = tuple(slice(None) + if d in unchanged_dims + else int_indexers.get(d, slice(None)) + for d in var.dims) + needs_masking = any(d in masked_dims for d in var.dims) + + if needs_masking: + new_var = var._getitem_with_mask(key) + elif all(is_full_slice(k) for k in key): + # no reindexing necessary # here we need to manually deal with copying data, since # we neither created a new ndarray nor used fancy indexing new_var = var.copy(deep=copy) + else: + new_var = var[key] reindexed[name] = new_var + return reindexed diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index ccbe48edc32..e811d189568 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -7,6 +7,17 @@ NA = utils.ReprObject('') +# Pairs of types that, if both found, should be promoted to object dtype +# instead of following NumPy's own type-promotion rules. These type promotion +# rules match pandas instead. For reference, see the NumPy type hierarchy: +# https://docs.scipy.org/doc/numpy-1.13.0/reference/arrays.scalars.html +PROMOTE_TO_OBJECT = [ + {np.number, np.character}, # numpy promotes to character + {np.bool_, np.character}, # numpy promotes to character + {np.bytes_, np.unicode_}, # numpy promotes to unicode +] + + def maybe_promote(dtype): """Simpler equivalent of pandas.core.common._maybe_promote @@ -60,3 +71,29 @@ def is_datetime_like(dtype): """ return (np.issubdtype(dtype, np.datetime64) or np.issubdtype(dtype, np.timedelta64)) + + +def result_type(*arrays_and_dtypes): + """Like np.result_type, but with type promotion rules matching pandas. + + Examples of changed behavior: + number + string -> object (not string) + bytes + unicode -> object (not unicode) + + Parameters + ---------- + *arrays_and_dtypes : list of arrays and dtypes + The dtype is extracted from both numpy and dask arrays. + + Returns + ------- + numpy.dtype for the result. + """ + types = {np.result_type(t).type for t in arrays_and_dtypes} + + for left, right in PROMOTE_TO_OBJECT: + if (any(issubclass(t, left) for t in types) and + any(issubclass(t, right) for t in types)): + return np.dtype(object) + + return np.result_type(*arrays_and_dtypes) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 2058ce86a99..3e9a2e6d154 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -82,13 +82,13 @@ def isnull(data): transpose = _dask_or_eager_func('transpose') -where = _dask_or_eager_func('where', n_array_args=3) +_where = _dask_or_eager_func('where', n_array_args=3) insert = _dask_or_eager_func('insert') take = _dask_or_eager_func('take') broadcast_to = _dask_or_eager_func('broadcast_to') -concatenate = _dask_or_eager_func('concatenate', list_of_args=True) -stack = _dask_or_eager_func('stack', list_of_args=True) +_concatenate = _dask_or_eager_func('concatenate', list_of_args=True) +_stack = _dask_or_eager_func('stack', list_of_args=True) array_all = _dask_or_eager_func('all') array_any = _dask_or_eager_func('any') @@ -100,6 +100,17 @@ def asarray(data): return data if isinstance(data, dask_array_type) else np.asarray(data) +def as_shared_dtype(scalars_or_arrays): + """Cast a arrays to a shared dtype using xarray's type promotion rules.""" + arrays = [asarray(x) for x in scalars_or_arrays] + # Pass arrays directly instead of dtypes to result_type so scalars + # get handled properly. + # Note that result_type() safely gets the dtype from dask arrays without + # evaluating them. + out_type = dtypes.result_type(*arrays) + return [x.astype(out_type, copy=False) for x in arrays] + + def as_like_arrays(*data): if all(isinstance(d, dask_array_type) for d in data): return data @@ -151,6 +162,11 @@ def count(data, axis=None): return sum(~isnull(data), axis=axis) +def where(condition, x, y): + """Three argument where() with better dtype promotion rules.""" + return _where(condition, *as_shared_dtype([x, y])) + + def where_method(data, cond, other=dtypes.NA): if other is dtypes.NA: other = dtypes.get_fill_value(data.dtype) @@ -161,6 +177,16 @@ def fillna(data, other): return where(isnull(data), other, data) +def concatenate(arrays, axis=0): + """concatenate() with better dtype promotion rules.""" + return _concatenate(as_shared_dtype(arrays), axis=axis) + + +def stack(arrays, axis=0): + """stack() with better dtype promotion rules.""" + return _stack(as_shared_dtype(arrays), axis=axis) + + @contextlib.contextmanager def _ignore_warnings_if(condition): if condition: diff --git a/xarray/core/variable.py b/xarray/core/variable.py index d4863014f59..14fc3480aa3 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1273,8 +1273,6 @@ def concat(cls, variables, dim='concat_dim', positions=None, arrays = [v.data for v in variables] - # TODO: use our own type promotion rules to ensure that - # [str, float] -> object, not str like numpy if dim in first_var.dims: axis = first_var.get_axis_num(dim) dims = first_var.dims diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 39b3109c295..0def5b6886e 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1717,6 +1717,12 @@ def test_where(self): actual = arr.where(arr.x < 2, drop=True) assert_identical(actual, expected) + def test_where_string(self): + array = DataArray(['a', 'b']) + expected = DataArray(np.array(['a', np.nan], dtype=object)) + actual = array.where([True, False]) + assert_identical(actual, expected) + def test_cumops(self): coords = {'x': [-1, -2], 'y': ['ab', 'cd', 'ef'], 'lat': (['x', 'y'], [[1, 2, 3], [-1, -2, -3]]), diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py new file mode 100644 index 00000000000..51c1aaa4c0c --- /dev/null +++ b/xarray/tests/test_dtypes.py @@ -0,0 +1,48 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import pytest + +from xarray.core import dtypes + + +@pytest.mark.parametrize("args, expected", [ + ([np.bool], np.bool), + ([np.bool, np.string_], np.object_), + ([np.float32, np.float64], np.float64), + ([np.float32, np.string_], np.object_), + ([np.unicode_, np.int64], np.object_), + ([np.unicode_, np.unicode_], np.unicode_), + ([np.bytes_, np.unicode_], np.object_), +]) +def test_result_type(args, expected): + actual = dtypes.result_type(*args) + assert actual == expected + + +def test_result_type_scalar(): + actual = dtypes.result_type(np.arange(3, dtype=np.float32), np.nan) + assert actual == np.float32 + + +def test_result_type_dask_array(): + # verify it works without evaluating dask arrays + da = pytest.importorskip('dask.array') + dask = pytest.importorskip('dask') + + def error(): + raise RuntimeError + + array = da.from_delayed(dask.delayed(error)(), (), np.float64) + with pytest.raises(RuntimeError): + array.compute() + + actual = dtypes.result_type(array) + assert actual == np.float64 + + # note that this differs from the behavior for scalar numpy arrays, which + # would get promoted to float32 + actual = dtypes.result_type(array, np.array([0.5, 1.0], dtype=np.float32)) + assert actual == np.float64 diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 9fb1b1aad40..54eaf46e054 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -6,7 +6,7 @@ from numpy import array, nan from . import assert_array_equal from xarray.core.duck_array_ops import ( - first, last, count, mean, array_notnull_equiv, + first, last, count, mean, array_notnull_equiv, where, stack, concatenate ) from . import TestCase, raises_regex @@ -76,6 +76,22 @@ def test_count(self): expected = array([[1, 2, 3], [3, 2, 1]]) assert_array_equal(expected, count(self.x, axis=-1)) + def test_where_type_promotion(self): + result = where([True, False], [1, 2], ['a', 'b']) + assert_array_equal(result, np.array([1, 'b'], dtype=object)) + + result = where([True, False], np.array([1, 2], np.float32), np.nan) + assert result.dtype == np.float32 + assert_array_equal(result, np.array([1, np.nan], dtype=np.float32)) + + def test_stack_type_promotion(self): + result = stack([1, 'b']) + assert_array_equal(result, np.array([1, 'b'], dtype=object)) + + def test_concatenate_type_promotion(self): + result = concatenate([[1], ['b']]) + assert_array_equal(result, np.array([1, 'b'], dtype=object)) + def test_all_nan_arrays(self): assert np.isnan(mean([np.nan, np.nan])) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 5a89627a0f9..d018df8abe4 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -468,10 +468,17 @@ def test_concat_number_strings(self): a = self.cls('x', ['0', '1', '2']) b = self.cls('x', ['3', '4']) actual = Variable.concat([a, b], dim='x') - expected = Variable('x', np.arange(5).astype(str).astype(object)) + expected = Variable('x', np.arange(5).astype(str)) assert_identical(expected, actual) - assert expected.dtype == object - assert type(expected.values[0]) == str + assert actual.dtype.kind == expected.dtype.kind + + def test_concat_mixed_dtypes(self): + a = self.cls('x', [0, 1]) + b = self.cls('x', ['two']) + actual = Variable.concat([a, b], dim='x') + expected = Variable('x', np.array([0, 1, 'two'], dtype=object)) + assert_identical(expected, actual) + assert actual.dtype == object def test_copy(self): v = self.cls('x', 0.5 * np.arange(10), {'foo': 'bar'}) From b6a0d60e720f5a19d6e00b11fc7f3d485e52a80c Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 16 Feb 2018 07:03:01 +0900 Subject: [PATCH 013/282] Support nan-ops for object-typed arrays (#1883) * First support of sum, min, max for object-typed arrays * typo * flake8 * Pandas compatiblity test. Added nanmean for object-type array * Improve test * Support nanvar, nanstd * Fix bug in _create_nan_agg_method * Added nanargmin/nanargmax * Support numpy<1.13. * Update tests. * Some cleanups and whatsnew * Simplify tests. Drop support std. * flake8 * xray -> xr * string array support * Support str dtype. Refactor nanmean * added get_pos_inifinity and get_neg_inifinity * Use function for get_fill_value instead of str. Add test to make sure it raises ValueError in argmin/argmax. * Tests for dtypes.INF --- doc/whats-new.rst | 11 +- xarray/core/dtypes.py | 87 +++++++++++++ xarray/core/duck_array_ops.py | 121 +++++++++++++++--- xarray/tests/test_dtypes.py | 6 + xarray/tests/test_duck_array_ops.py | 184 +++++++++++++++++++++++++++- 5 files changed, 384 insertions(+), 25 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0a3f949b496..bc5a5bb5ea4 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -43,7 +43,16 @@ Documentation Enhancements ~~~~~~~~~~~~ -- reduce methods such as :py:func:`DataArray.sum()` now accepts ``dtype`` +- Reduce methods such as :py:func:`DataArray.sum()` now handles object-type array. + + .. ipython:: python + + da = xr.DataArray(np.array([True, False, np.nan], dtype=object), dims='x') + da.sum() + + (:issue:`1866`) + By `Keisuke Fujii `_. +- Reduce methods such as :py:func:`DataArray.sum()` now accepts ``dtype`` arguments. (:issue:`1838`) By `Keisuke Fujii `_. - Added nodatavals attribute to DataArray when using :py:func:`~xarray.open_rasterio`. (:issue:`1736`). diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index e811d189568..8dac39612e4 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -1,4 +1,5 @@ import numpy as np +import functools from . import utils @@ -7,6 +8,29 @@ NA = utils.ReprObject('') +@functools.total_ordering +class AlwaysGreaterThan(object): + def __gt__(self, other): + return True + + def __eq__(self, other): + return isinstance(other, type(self)) + + +@functools.total_ordering +class AlwaysLessThan(object): + def __lt__(self, other): + return True + + def __eq__(self, other): + return isinstance(other, type(self)) + + +# Equivalence to np.inf (-np.inf) for object-type +INF = AlwaysGreaterThan() +NINF = AlwaysLessThan() + + # Pairs of types that, if both found, should be promoted to object dtype # instead of following NumPy's own type-promotion rules. These type promotion # rules match pandas instead. For reference, see the NumPy type hierarchy: @@ -18,6 +42,29 @@ ] +@functools.total_ordering +class AlwaysGreaterThan(object): + def __gt__(self, other): + return True + + def __eq__(self, other): + return isinstance(other, type(self)) + + +@functools.total_ordering +class AlwaysLessThan(object): + def __lt__(self, other): + return True + + def __eq__(self, other): + return isinstance(other, type(self)) + + +# Equivalence to np.inf (-np.inf) for object-type +INF = AlwaysGreaterThan() +NINF = AlwaysLessThan() + + def maybe_promote(dtype): """Simpler equivalent of pandas.core.common._maybe_promote @@ -66,6 +113,46 @@ def get_fill_value(dtype): return fill_value +def get_pos_infinity(dtype): + """Return an appropriate positive infinity for this dtype. + + Parameters + ---------- + dtype : np.dtype + + Returns + ------- + fill_value : positive infinity value corresponding to this dtype. + """ + if issubclass(dtype.type, (np.floating, np.integer)): + return np.inf + + if issubclass(dtype.type, np.complexfloating): + return np.inf + 1j * np.inf + + return INF + + +def get_neg_infinity(dtype): + """Return an appropriate positive infinity for this dtype. + + Parameters + ---------- + dtype : np.dtype + + Returns + ------- + fill_value : positive infinity value corresponding to this dtype. + """ + if issubclass(dtype.type, (np.floating, np.integer)): + return -np.inf + + if issubclass(dtype.type, np.complexfloating): + return -np.inf - 1j * np.inf + + return NINF + + def is_datetime_like(dtype): """Check if a dtype is a subclass of the numpy datetime types """ diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 3e9a2e6d154..6f5548800a2 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -197,6 +197,79 @@ def _ignore_warnings_if(condition): yield +def _nansum_object(value, axis=None, **kwargs): + """ In house nansum for object array """ + value = fillna(value, 0) + return _dask_or_eager_func('sum')(value, axis=axis, **kwargs) + + +def _nan_minmax_object(func, get_fill_value, value, axis=None, **kwargs): + """ In house nanmin and nanmax for object array """ + fill_value = get_fill_value(value.dtype) + valid_count = count(value, axis=axis) + filled_value = fillna(value, fill_value) + data = _dask_or_eager_func(func)(filled_value, axis=axis, **kwargs) + if not hasattr(data, 'dtype'): # scalar case + data = dtypes.fill_value(value.dtype) if valid_count == 0 else data + return np.array(data, dtype=value.dtype) + return where_method(data, valid_count != 0) + + +def _nan_argminmax_object(func, get_fill_value, value, axis=None, **kwargs): + """ In house nanargmin, nanargmax for object arrays. Always return integer + type """ + fill_value = get_fill_value(value.dtype) + valid_count = count(value, axis=axis) + value = fillna(value, fill_value) + data = _dask_or_eager_func(func)(value, axis=axis, **kwargs) + # dask seems return non-integer type + if isinstance(value, dask_array_type): + data = data.astype(int) + + if (valid_count == 0).any(): + raise ValueError('All-NaN slice encountered') + + return np.array(data, dtype=int) + + +def _nanmean_ddof_object(ddof, value, axis=None, **kwargs): + """ In house nanmean. ddof argument will be used in _nanvar method """ + valid_count = count(value, axis=axis) + value = fillna(value, 0) + # As dtype inference is impossible for object dtype, we assume float + # https://github.com/dask/dask/issues/3162 + dtype = kwargs.pop('dtype', None) + if dtype is None and value.dtype.kind == 'O': + dtype = value.dtype if value.dtype.kind in ['cf'] else float + + data = _dask_or_eager_func('sum')(value, axis=axis, dtype=dtype, **kwargs) + data = data / (valid_count - ddof) + return where_method(data, valid_count != 0) + + +def _nanvar_object(value, axis=None, **kwargs): + ddof = kwargs.pop('ddof', 0) + kwargs_mean = kwargs.copy() + kwargs_mean.pop('keepdims', None) + value_mean = _nanmean_ddof_object(ddof=0, value=value, axis=axis, + keepdims=True, **kwargs_mean) + squared = (value.astype(value_mean.dtype) - value_mean)**2 + return _nanmean_ddof_object(ddof, squared, axis=axis, **kwargs) + + +_nan_object_funcs = { + 'sum': _nansum_object, + 'min': partial(_nan_minmax_object, 'min', dtypes.get_pos_infinity), + 'max': partial(_nan_minmax_object, 'max', dtypes.get_neg_infinity), + 'argmin': partial(_nan_argminmax_object, 'argmin', + dtypes.get_pos_infinity), + 'argmax': partial(_nan_argminmax_object, 'argmax', + dtypes.get_neg_infinity), + 'mean': partial(_nanmean_ddof_object, 0), + 'var': _nanvar_object, +} + + def _create_nan_agg_method(name, numeric_only=False, np_compat=False, no_bottleneck=False, coerce_strings=False, keep_dims=False): @@ -211,27 +284,31 @@ def f(values, axis=None, skipna=None, **kwargs): if coerce_strings and values.dtype.kind in 'SU': values = values.astype(object) - if skipna or (skipna is None and values.dtype.kind in 'cf'): + if skipna or (skipna is None and values.dtype.kind in 'cfO'): if values.dtype.kind not in ['u', 'i', 'f', 'c']: - raise NotImplementedError( - 'skipna=True not yet implemented for %s with dtype %s' - % (name, values.dtype)) - nanname = 'nan' + name - if (isinstance(axis, tuple) or not values.dtype.isnative or - no_bottleneck or - (dtype is not None and np.dtype(dtype) != values.dtype)): - # bottleneck can't handle multiple axis arguments or non-native - # endianness - if np_compat: - eager_module = npcompat - else: - eager_module = np + func = _nan_object_funcs.get(name, None) + using_numpy_nan_func = True + if func is None or values.dtype.kind not in 'Ob': + raise NotImplementedError( + 'skipna=True not yet implemented for %s with dtype %s' + % (name, values.dtype)) else: - kwargs.pop('dtype', None) - eager_module = bn - func = _dask_or_eager_func(nanname, eager_module) - using_numpy_nan_func = (eager_module is np or - eager_module is npcompat) + nanname = 'nan' + name + if (isinstance(axis, tuple) or not values.dtype.isnative or + no_bottleneck or (dtype is not None and + np.dtype(dtype) != values.dtype)): + # bottleneck can't handle multiple axis arguments or + # non-native endianness + if np_compat: + eager_module = npcompat + else: + eager_module = np + else: + kwargs.pop('dtype', None) + eager_module = bn + func = _dask_or_eager_func(nanname, eager_module) + using_numpy_nan_func = (eager_module is np or + eager_module is npcompat) else: func = _dask_or_eager_func(name) using_numpy_nan_func = False @@ -240,7 +317,11 @@ def f(values, axis=None, skipna=None, **kwargs): return func(values, axis=axis, **kwargs) except AttributeError: if isinstance(values, dask_array_type): - msg = '%s is not yet implemented on dask arrays' % name + try: # dask/dask#3133 dask sometimes needs dtype argument + return func(values, axis=axis, dtype=values.dtype, + **kwargs) + except AttributeError: + msg = '%s is not yet implemented on dask arrays' % name else: assert using_numpy_nan_func msg = ('%s is not available with skipna=False with the ' diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 51c1aaa4c0c..1b236e0160d 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -46,3 +46,9 @@ def error(): # would get promoted to float32 actual = dtypes.result_type(array, np.array([0.5, 1.0], dtype=np.float32)) assert actual == np.float64 + + +@pytest.mark.parametrize('obj', [1.0, np.inf, 'ab', 1.0 + 1.0j, True]) +def test_inf(obj): + assert dtypes.INF > obj + assert dtypes.NINF < obj diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 54eaf46e054..d68a7a382de 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -1,15 +1,19 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from pytest import mark +import pytest import numpy as np from numpy import array, nan +from distutils.version import LooseVersion from . import assert_array_equal from xarray.core.duck_array_ops import ( first, last, count, mean, array_notnull_equiv, where, stack, concatenate ) +from xarray import DataArray +from xarray.testing import assert_allclose +from xarray import concat -from . import TestCase, raises_regex +from . import TestCase, raises_regex, has_dask class TestOps(TestCase): @@ -97,7 +101,7 @@ def test_all_nan_arrays(self): class TestArrayNotNullEquiv(): - @mark.parametrize("arr1, arr2", [ + @pytest.mark.parametrize("arr1, arr2", [ (np.array([1, 2, 3]), np.array([1, 2, 3])), (np.array([1, 2, np.nan]), np.array([1, np.nan, 3])), (np.array([np.nan, 2, np.nan]), np.array([1, np.nan, np.nan])), @@ -115,7 +119,7 @@ def test_wrong_shape(self): b = np.array([[1, 2], [np.nan, 4]]) assert not array_notnull_equiv(a, b) - @mark.parametrize("val1, val2, val3, null", [ + @pytest.mark.parametrize("val1, val2, val3, null", [ (1, 2, 3, None), (1., 2., 3., np.nan), (1., 2., 3., None), @@ -125,3 +129,175 @@ def test_types(self, val1, val2, val3, null): arr1 = np.array([val1, null, val3, null]) arr2 = np.array([val1, val2, null, null]) assert array_notnull_equiv(arr1, arr2) + + +def construct_dataarray(dim_num, dtype, contains_nan, dask): + # dimnum <= 3 + rng = np.random.RandomState(0) + shapes = [16, 8, 4][:dim_num] + dims = ('x', 'y', 'z')[:dim_num] + + if np.issubdtype(dtype, np.floating): + array = rng.randn(*shapes).astype(dtype) + elif np.issubdtype(dtype, np.integer): + array = rng.randint(0, 10, size=shapes).astype(dtype) + elif np.issubdtype(dtype, np.bool_): + array = rng.randint(0, 1, size=shapes).astype(dtype) + elif dtype == str: + array = rng.choice(['a', 'b', 'c', 'd'], size=shapes) + else: + raise ValueError + da = DataArray(array, dims=dims, coords={'x': np.arange(16)}, name='da') + + if contains_nan: + da = da.reindex(x=np.arange(20)) + if dask and has_dask: + chunks = {d: 4 for d in dims} + da = da.chunk(chunks) + + return da + + +def from_series_or_scalar(se): + try: + return DataArray.from_series(se) + except AttributeError: # scalar case + return DataArray(se) + + +def series_reduce(da, func, dim, **kwargs): + """ convert DataArray to pd.Series, apply pd.func, then convert back to + a DataArray. Multiple dims cannot be specified.""" + if dim is None or da.ndim == 1: + se = da.to_series() + return from_series_or_scalar(getattr(se, func)(**kwargs)) + else: + da1 = [] + dims = list(da.dims) + dims.remove(dim) + d = dims[0] + for i in range(len(da[d])): + da1.append(series_reduce(da.isel(**{d: i}), func, dim, **kwargs)) + + if d in da.coords: + return concat(da1, dim=da[d]) + return concat(da1, dim=d) + + +@pytest.mark.parametrize('dim_num', [1, 2]) +@pytest.mark.parametrize('dtype', [float, int, np.float32, np.bool_]) +@pytest.mark.parametrize('dask', [False, True]) +@pytest.mark.parametrize('func', ['sum', 'min', 'max', 'mean', 'var']) +@pytest.mark.parametrize('skipna', [False, True]) +@pytest.mark.parametrize('aggdim', [None, 'x']) +def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): + + if aggdim == 'y' and dim_num < 2: + return + + if dtype == np.bool_ and func == 'mean': + return # numpy does not support this + + if dask and not has_dask: + return + + rtol = 1e-04 if dtype == np.float32 else 1e-05 + + da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask) + axis = None if aggdim is None else da.get_axis_num(aggdim) + + if dask and not skipna and func in ['var', 'std'] and dtype == np.bool_: + # TODO this might be dask's bug + return + + if (LooseVersion(np.__version__) >= LooseVersion('1.13.0') and + da.dtype.kind == 'O' and skipna): + # Numpy < 1.13 does not handle object-type array. + try: + if skipna: + expected = getattr(np, 'nan{}'.format(func))(da.values, + axis=axis) + else: + expected = getattr(np, func)(da.values, axis=axis) + + actual = getattr(da, func)(skipna=skipna, dim=aggdim) + assert np.allclose(actual.values, np.array(expected), rtol=1.0e-4, + equal_nan=True) + except (TypeError, AttributeError, ZeroDivisionError): + # TODO currently, numpy does not support some methods such as + # nanmean for object dtype + pass + + # make sure the compatiblility with pandas' results. + actual = getattr(da, func)(skipna=skipna, dim=aggdim) + if func == 'var': + expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=0) + assert_allclose(actual, expected, rtol=rtol) + # also check ddof!=0 case + actual = getattr(da, func)(skipna=skipna, dim=aggdim, ddof=5) + expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=5) + assert_allclose(actual, expected, rtol=rtol) + else: + expected = series_reduce(da, func, skipna=skipna, dim=aggdim) + assert_allclose(actual, expected, rtol=rtol) + + # make sure the dtype argument + if func not in ['max', 'min']: + actual = getattr(da, func)(skipna=skipna, dim=aggdim, dtype=float) + assert actual.dtype == float + + # without nan + da = construct_dataarray(dim_num, dtype, contains_nan=False, dask=dask) + actual = getattr(da, func)(skipna=skipna) + expected = getattr(np, 'nan{}'.format(func))(da.values) + if actual.dtype == object: + assert actual.values == np.array(expected) + else: + assert np.allclose(actual.values, np.array(expected), rtol=rtol) + + +@pytest.mark.parametrize('dim_num', [1, 2]) +@pytest.mark.parametrize('dtype', [float, int, np.float32, np.bool_, str]) +@pytest.mark.parametrize('contains_nan', [True, False]) +@pytest.mark.parametrize('dask', [False, True]) +@pytest.mark.parametrize('func', ['min', 'max']) +@pytest.mark.parametrize('skipna', [False, True]) +@pytest.mark.parametrize('aggdim', ['x', 'y']) +def test_argmin_max(dim_num, dtype, contains_nan, dask, func, skipna, aggdim): + # pandas-dev/pandas#16830, we do not check consistency with pandas but + # just make sure da[da.argmin()] == da.min() + + if aggdim == 'y' and dim_num < 2: + return + + if dask and not has_dask: + return + + if contains_nan: + if not skipna: + # numpy's argmin (not nanargmin) does not handle object-dtype + return + if skipna and np.dtype(dtype).kind in 'iufc': + # numpy's nanargmin raises ValueError for all nan axis + return + + da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, + dask=dask) + + if aggdim == 'y' and contains_nan and skipna: + with pytest.raises(ValueError): + actual = da.isel(**{ + aggdim: getattr(da, 'arg'+func)(dim=aggdim, + skipna=skipna).compute()}) + return + + actual = da.isel(**{ + aggdim: getattr(da, 'arg'+func)(dim=aggdim, skipna=skipna).compute()}) + expected = getattr(da, func)(dim=aggdim, skipna=skipna) + assert_allclose(actual.drop(actual.coords), expected.drop(expected.coords)) + + +def test_argmin_max_error(): + da = construct_dataarray(2, np.bool_, contains_nan=True, dask=False) + with pytest.raises(ValueError): + da.argmin(dim='y') From 174fe5d28b94147ecb0cb88d5301ca9479c6d7ad Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 15 Feb 2018 15:20:30 -0800 Subject: [PATCH 014/282] Build documentation on TravisCI (#1908) * test sphinx doc build on travis * test_env name for all environments * don't run unit tests on doc build * fix travis yaml * semicolons * sphinx_rtd_theme * install theme after the activating environment * fix list formatting in installing.rst * fail travis doc build only when full errors (not warnings) are found * flake8 for docs build --- .travis.yml | 17 ++++++++++++++--- doc/environment.yml | 1 + doc/installing.rst | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 068ea3cc788..ee8ffcc4d5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,8 @@ matrix: env: CONDA_ENV=py36-rasterio1.0alpha - python: 3.6 env: CONDA_ENV=py36-zarr-dev + - python: 3.5 + env: CONDA_ENV=docs allow_failures: - python: 3.6 env: @@ -86,16 +88,25 @@ before_install: - conda info -a install: - - conda env create --file ci/requirements-$CONDA_ENV.yml + - if [[ "$CONDA_ENV" == "docs" ]]; then + conda env create -n test_env --file doc/environment.yml; + else + conda env create -n test_env --file ci/requirements-$CONDA_ENV.yml; + fi - source activate test_env - conda list - - python setup.py install + - pip install --no-deps -e . - python xarray/util/print_versions.py script: - flake8 -j auto xarray - python -OO -c "import xarray" - - py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing --verbose $EXTRA_FLAGS + - if [[ "$CONDA_ENV" == "docs" ]]; then + conda install -c conda-forge sphinx_rtd_theme; + sphinx-build -n -b html -d _build/doctrees doc _build/html; + else + py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing --verbose $EXTRA_FLAGS; + fi after_success: - coveralls diff --git a/doc/environment.yml b/doc/environment.yml index b14fba351c1..2758612c139 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -18,3 +18,4 @@ dependencies: - sphinx-gallery - zarr - iris + - flake8 diff --git a/doc/installing.rst b/doc/installing.rst index b6ec2bd841f..b9a1fff59cc 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -78,6 +78,7 @@ Testing ------- To run the test suite after installing xarray, first install (via pypi or conda) + - `py.test `__: Simple unit testing library - `mock `__: additional testing library required for python version 2 From d191352b6c1e15a2b6105b4b76552fe974231396 Mon Sep 17 00:00:00 2001 From: Stickler Bot Date: Thu, 15 Feb 2018 23:21:28 +0000 Subject: [PATCH 015/282] Adding .stickler.yml configuration file (#1913) * Adding .stickler.yml * Add max line length and py3k --- .stickler.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .stickler.yml diff --git a/.stickler.yml b/.stickler.yml new file mode 100644 index 00000000000..1634c1f53c8 --- /dev/null +++ b/.stickler.yml @@ -0,0 +1,7 @@ +linters: + flake8: + max-line-length: 79 + fixer: true + py3k: +fixers: + enable: true From 58bc0240b51addd051d97e92ef2e29a9de5b92b0 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Feb 2018 22:02:57 +0100 Subject: [PATCH 016/282] COMPAT: MultiIndex checking is fragile (#1833) (#1916) --- doc/whats-new.rst | 3 +++ xarray/core/dataset.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index bc5a5bb5ea4..fd0e6b9f28b 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -492,6 +492,9 @@ Bug fixes ``apionly`` module was deprecated. (:issue:`1633`). By `Joe Hamman `_. +- Fix COMPAT: MultiIndex checking is fragile + (:issue:`1833`). By `Florian Pinault `_. + - Fix ``rasterio`` backend for Rasterio versions 1.0alpha10 and newer. (:issue:`1641`). By `Chris Holden `_. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 62ad2b9b653..98b0b055d9c 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -2752,7 +2752,7 @@ def from_dataframe(cls, dataframe): idx = dataframe.index obj = cls() - if hasattr(idx, 'levels'): + if isinstance(idx, pd.MultiIndex): # it's a multi-index # expand the DataFrame to include the product of all levels full_idx = pd.MultiIndex.from_product(idx.levels, names=idx.names) From 8c5c549e842644b3b5acabf40a98d7fe7e33ad89 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Fri, 16 Feb 2018 22:08:31 +0100 Subject: [PATCH 017/282] _color_palette consistent whith or without seaborn (#1902) --- doc/whats-new.rst | 3 +++ xarray/plot/utils.py | 31 +++++++++++-------------------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fd0e6b9f28b..fe8332c311b 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -152,6 +152,9 @@ Bug fixes ``parse_coordinates`` kwarg has beed added to :py:func:`~open_rasterio` (set to ``True`` per default). By `Fabien Maussion `_. +- The colors of discrete colormaps are now the same regardless if `seaborn` + is installed or not (:issue:`1896`). + By `Fabien Maussion `_. - Fixed dtype promotion rules in :py:func:`where` and :py:func:`concat` to match pandas (:issue:`1847`). A combination of strings/numbers or unicode/bytes now promote to object dtype, instead of strings or unicode. diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index c194b9dd8d8..0e565f24a60 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -115,32 +115,23 @@ def _color_palette(cmap, n_colors): colors_i = np.linspace(0, 1., n_colors) if isinstance(cmap, (list, tuple)): # we have a list of colors - try: - sns = import_seaborn() - except ImportError: - # if that fails, use matplotlib - # in this case, is there any difference between mpl and seaborn? - cmap = ListedColormap(cmap, N=n_colors) - pal = cmap(colors_i) - else: - # first try to turn it into a palette with seaborn - pal = sns.color_palette(cmap, n_colors=n_colors) + cmap = ListedColormap(cmap, N=n_colors) + pal = cmap(colors_i) elif isinstance(cmap, basestring): # we have some sort of named palette try: - # first try to turn it into a palette with seaborn - from seaborn.apionly import color_palette - pal = color_palette(cmap, n_colors=n_colors) - except (ImportError, ValueError): - # ValueError is raised when seaborn doesn't like a colormap - # (e.g. jet). If that fails, use matplotlib + # is this a matplotlib cmap? + cmap = plt.get_cmap(cmap) + pal = cmap(colors_i) + except ValueError: + # ValueError happens when mpl doesn't like a colormap, try seaborn try: - # is this a matplotlib cmap? - cmap = plt.get_cmap(cmap) - except ValueError: + from seaborn.apionly import color_palette + pal = color_palette(cmap, n_colors=n_colors) + except (ValueError, ImportError): # or maybe we just got a single color as a string cmap = ListedColormap([cmap], N=n_colors) - pal = cmap(colors_i) + pal = cmap(colors_i) else: # cmap better be a LinearSegmentedColormap (e.g. viridis) pal = cmap(colors_i) From e0621c7d66c13b486b1890f67a126caec2990da7 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 16 Feb 2018 20:40:15 -0800 Subject: [PATCH 018/282] drop zarr variable name from the dask chunk name (#1907) --- xarray/backends/zarr.py | 2 +- xarray/core/dataset.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 02753f6cca9..2737d9fb213 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -482,7 +482,7 @@ def maybe_chunk(name, var): if (var.ndim > 0) and (chunks is not None): # does this cause any data to be read? token2 = tokenize(name, var._data) - name2 = 'zarr-%s-%s' % (name, token2) + name2 = 'zarr-%s' % token2 return var.chunk(chunks, name=name2, lock=None) else: return var diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 98b0b055d9c..5f31a3d9483 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1253,7 +1253,7 @@ def chunk(self, chunks=None, name_prefix='xarray-', token=None, from dask.base import tokenize except ImportError: import dask # raise the usual error if dask is entirely missing # flake8: noqa - raise ImportError('xarray requires dask version 0.6 or newer') + raise ImportError('xarray requires dask version 0.9 or newer') if isinstance(chunks, Number): chunks = dict.fromkeys(self.dims, chunks) From 2ff7b4c4e394bfe73445f8cf471f0df8b79417bf Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sun, 18 Feb 2018 16:26:29 +0900 Subject: [PATCH 019/282] Support indexing with 0d-np.ndarray (#1922) --- doc/whats-new.rst | 2 ++ xarray/core/variable.py | 6 +++++- xarray/tests/test_variable.py | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fe8332c311b..6018a184875 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -106,6 +106,8 @@ Enhancements Bug fixes ~~~~~~~~~ +- Support indexing with a 0d-np.ndarray (:issue:`1921`). + By `Keisuke Fujii `_. - Added warning in api.py of a netCDF4 bug that occurs when the filepath has 88 characters (:issue:`1745`). By `Liam Brannigan `_. diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 14fc3480aa3..267dc02ce13 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -463,10 +463,14 @@ def _broadcast_indexes(self, key): key = self._item_key_to_tuple(key) # key is a tuple # key is a tuple of full size key = indexing.expanded_indexer(key, self.ndim) - # Convert a scalar Variable as an integer + # Convert a scalar Variable to an integer key = tuple( k.data.item() if isinstance(k, Variable) and k.ndim == 0 else k for k in key) + # Convert a 0d-array to an integer + key = tuple( + k.item() if isinstance(k, np.ndarray) and k.ndim == 0 else k + for k in key) if all(isinstance(k, BASIC_INDEXING_TYPES) for k in key): return self._broadcast_indexes_basic(key) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index d018df8abe4..c8f74762683 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -627,6 +627,12 @@ def test_getitem_0d_array(self): v_new = v[np.array([0])[0]] assert_array_equal(v_new, v_data[0]) + v_new = v[np.array(0)] + assert_array_equal(v_new, v_data[0]) + + v_new = v[Variable((), np.array(0))] + assert_array_equal(v_new, v_data[0]) + def test_getitem_fancy(self): v = self.cls(['x', 'y'], [[0, 1, 2], [3, 4, 5]]) v_data = v.compute().data From be27319bcf6a754f1655918fac45c1cd2cfa0cec Mon Sep 17 00:00:00 2001 From: Noah D Brenowitz Date: Sun, 18 Feb 2018 11:06:30 -0800 Subject: [PATCH 020/282] Warn when pcolormesh coordinate is not sorted (#1885) Raise when pcolormesh coordinate is not sorted --- xarray/plot/plot.py | 29 +++++++++++++++++++++++++++++ xarray/tests/test_plot.py | 4 ++++ 2 files changed, 33 insertions(+) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 97fee5f01c0..162a0c238f5 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -745,6 +745,26 @@ def contourf(x, y, z, ax, **kwargs): return primitive +def _is_monotonic(coord, axis=0): + """ + >>> _is_monotonic(np.array([0, 1, 2])) + True + >>> _is_monotonic(np.array([2, 1, 0])) + True + >>> _is_monotonic(np.array([0, 2, 1])) + False + """ + if coord.shape[axis] < 3: + return True + else: + n = coord.shape[axis] + delta_pos = (coord.take(np.arange(1, n), axis=axis) >= + coord.take(np.arange(0, n-1), axis=axis)) + delta_neg = (coord.take(np.arange(1, n), axis=axis) <= + coord.take(np.arange(0, n-1), axis=axis)) + return np.all(delta_pos) or np.all(delta_neg) + + def _infer_interval_breaks(coord, axis=0): """ >>> _infer_interval_breaks(np.arange(5)) @@ -754,6 +774,15 @@ def _infer_interval_breaks(coord, axis=0): [ 2.5, 3.5, 4.5]]) """ coord = np.asarray(coord) + + if not _is_monotonic(coord, axis=axis): + raise ValueError("The input coordinate is not sorted in increasing " + "order along axis %d. This can lead to unexpected " + "results. Consider calling the `sortby` method on " + "the input DataArray. To plot data with categorical " + "axes, consider using the `heatmap` function from " + "the `seaborn` statistical plotting library." % axis) + deltas = 0.5 * np.diff(coord, axis=axis) if deltas.size == 0: deltas = np.array(0.0) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 479b1dce9da..46410cd53e3 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -178,6 +178,10 @@ def test__infer_interval_breaks(self): np.testing.assert_allclose(xref, x) np.testing.assert_allclose(yref, y) + # test that warning is raised for non-monotonic inputs + with pytest.raises(ValueError): + _infer_interval_breaks(np.array([0, 2, 1])) + def test_datetime_dimension(self): nrow = 3 ncol = 4 From e544e0db42819076dc4a0c70f513b726fe8da511 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 19 Feb 2018 13:25:56 -0800 Subject: [PATCH 021/282] Add netcdftime as an optional dependency. (#1920) * rework imports to support using netcdftime package when netcdf4-python is not installed * temporary travis build for netcdftime dev * flake8 and rework import logic * add missing netcdftime environment * fix typo in unidata * cython too * use conda-forge * require netcdf4 for tests that read/write --- .travis.yml | 4 ++ ci/requirements-py36-netcdftime-dev.yml | 13 ++++++ doc/installing.rst | 2 + doc/whats-new.rst | 4 ++ xarray/backends/api.py | 2 +- xarray/coding/times.py | 57 ++++++++++++++++--------- xarray/tests/__init__.py | 1 + xarray/tests/test_conventions.py | 12 +++--- 8 files changed, 69 insertions(+), 26 deletions(-) create mode 100644 ci/requirements-py36-netcdftime-dev.yml diff --git a/.travis.yml b/.travis.yml index ee8ffcc4d5e..70c0a63ae08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,8 @@ matrix: env: CONDA_ENV=py36-rasterio1.0alpha - python: 3.6 env: CONDA_ENV=py36-zarr-dev + - python: 3.6 + env: CONDA_ENV=py36-netcdftime-dev - python: 3.5 env: CONDA_ENV=docs allow_failures: @@ -73,6 +75,8 @@ matrix: env: CONDA_ENV=py36-rasterio1.0alpha - python: 3.6 env: CONDA_ENV=py36-zarr-dev + - python: 3.6 + env: CONDA_ENV=py36-netcdftime-dev before_install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then diff --git a/ci/requirements-py36-netcdftime-dev.yml b/ci/requirements-py36-netcdftime-dev.yml new file mode 100644 index 00000000000..5c2193474b4 --- /dev/null +++ b/ci/requirements-py36-netcdftime-dev.yml @@ -0,0 +1,13 @@ +name: test_env +channels: + - conda-forge +dependencies: + - python=3.6 + - pytest + - flake8 + - numpy + - pandas + - netcdftime + - pip: + - coveralls + - pytest-cov diff --git a/doc/installing.rst b/doc/installing.rst index b9a1fff59cc..8be025665e2 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -25,6 +25,8 @@ For netCDF and IO - `pynio `__: for reading GRIB and other geoscience specific file formats - `zarr `__: for chunked, compressed, N-dimensional arrays. +- `netcdftime `__: recommended if you + want to encode/decode datetimes for non-standard calendars. For accelerating xarray ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 6018a184875..42c0891f1a6 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -93,6 +93,10 @@ Enhancements - Speed of reindexing/alignment with dask array is orders of magnitude faster when inserting missing values (:issue:`1847`). By `Stephan Hoyer `_. +- Add ``netcdftime`` as an optional dependency of xarray. This allows for + encoding/decoding of datetimes with non-standard calendars without the + netCDF4 dependency (:issue:`1084`). + By `Joe Hamman `_. .. _Zarr: http://zarr.readthedocs.io/ diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 668fb53899d..1effdf18dac 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -443,7 +443,7 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, lock=None, data_vars='all', coords='different', **kwargs): """Open multiple files as a single dataset. - Requires dask to be installed. See documentation for details on dask [1]. + Requires dask to be installed. See documentation for details on dask [1]. Attributes from the first dataset file are used for the combined dataset. Parameters diff --git a/xarray/coding/times.py b/xarray/coding/times.py index e00769af884..28afc46f660 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -40,6 +40,26 @@ 'milliseconds', 'microseconds']) +def _import_netcdftime(): + ''' + helper function handle the transition to netcdftime as a stand-alone + package + ''' + try: + # Try importing netcdftime directly + import netcdftime as nctime + if not hasattr(nctime, 'num2date'): + # must have gotten an old version from netcdf4-python + raise ImportError + except ImportError: + # in netCDF4 the num2date/date2num function are top-level api + try: + import netCDF4 as nctime + except ImportError: + raise ImportError("Failed to import netcdftime") + return nctime + + def _netcdf_to_numpy_timeunit(units): units = units.lower() if not units.endswith('s'): @@ -59,15 +79,15 @@ def _unpack_netcdf_time_units(units): return delta_units, ref_date -def _decode_datetime_with_netcdf4(num_dates, units, calendar): - import netCDF4 as nc4 +def _decode_datetime_with_netcdftime(num_dates, units, calendar): + nctime = _import_netcdftime() - dates = np.asarray(nc4.num2date(num_dates, units, calendar)) + dates = np.asarray(nctime.num2date(num_dates, units, calendar)) if (dates[np.nanargmin(num_dates)].year < 1678 or dates[np.nanargmax(num_dates)].year >= 2262): warnings.warn('Unable to decode time axis into full ' 'numpy.datetime64 objects, continuing using dummy ' - 'netCDF4.datetime objects instead, reason: dates out' + 'netcdftime.datetime objects instead, reason: dates out' ' of range', SerializationWarning, stacklevel=3) else: try: @@ -75,7 +95,7 @@ def _decode_datetime_with_netcdf4(num_dates, units, calendar): except ValueError as e: warnings.warn('Unable to decode time axis into full ' 'numpy.datetime64 objects, continuing using ' - 'dummy netCDF4.datetime objects instead, reason:' + 'dummy netcdftime.datetime objects instead, reason:' '{0}'.format(e), SerializationWarning, stacklevel=3) return dates @@ -111,7 +131,7 @@ def decode_cf_datetime(num_dates, units, calendar=None): numpy array of date time objects. For standard (Gregorian) calendars, this function uses vectorized - operations, which makes it much faster than netCDF4.num2date. In such a + operations, which makes it much faster than netcdftime.num2date. In such a case, the returned array will be of type np.datetime64. Note that time unit in `units` must not be smaller than microseconds and @@ -119,7 +139,7 @@ def decode_cf_datetime(num_dates, units, calendar=None): See also -------- - netCDF4.num2date + netcdftime.num2date """ num_dates = np.asarray(num_dates) flat_num_dates = num_dates.ravel() @@ -137,7 +157,7 @@ def decode_cf_datetime(num_dates, units, calendar=None): ref_date = pd.Timestamp(ref_date) except ValueError: # ValueError is raised by pd.Timestamp for non-ISO timestamp - # strings, in which case we fall back to using netCDF4 + # strings, in which case we fall back to using netcdftime raise OutOfBoundsDatetime # fixes: https://github.com/pydata/pandas/issues/14068 @@ -155,9 +175,8 @@ def decode_cf_datetime(num_dates, units, calendar=None): ref_date).values except (OutOfBoundsDatetime, OverflowError): - dates = _decode_datetime_with_netcdf4(flat_num_dates.astype(np.float), - units, - calendar) + dates = _decode_datetime_with_netcdftime( + flat_num_dates.astype(np.float), units, calendar) return dates.reshape(num_dates.shape) @@ -215,7 +234,7 @@ def infer_timedelta_units(deltas): def nctime_to_nptime(times): - """Given an array of netCDF4.datetime objects, return an array of + """Given an array of netcdftime.datetime objects, return an array of numpy.datetime64 objects of the same size""" times = np.asarray(times) new = np.empty(times.shape, dtype='M8[ns]') @@ -235,20 +254,20 @@ def _cleanup_netcdf_time_units(units): return units -def _encode_datetime_with_netcdf4(dates, units, calendar): - """Fallback method for encoding dates using netCDF4-python. +def _encode_datetime_with_netcdftime(dates, units, calendar): + """Fallback method for encoding dates using netcdftime. This method is more flexible than xarray's parsing using datetime64[ns] arrays but also slower because it loops over each element. """ - import netCDF4 as nc4 + nctime = _import_netcdftime() if np.issubdtype(dates.dtype, np.datetime64): # numpy's broken datetime conversion only works for us precision dates = dates.astype('M8[us]').astype(datetime) def encode_datetime(d): - return np.nan if d is None else nc4.date2num(d, units, calendar) + return np.nan if d is None else nctime.date2num(d, units, calendar) return np.vectorize(encode_datetime)(dates) @@ -268,7 +287,7 @@ def encode_cf_datetime(dates, units=None, calendar=None): See also -------- - netCDF4.date2num + netcdftime.date2num """ dates = np.asarray(dates) @@ -283,7 +302,7 @@ def encode_cf_datetime(dates, units=None, calendar=None): delta, ref_date = _unpack_netcdf_time_units(units) try: if calendar not in _STANDARD_CALENDARS or dates.dtype.kind == 'O': - # parse with netCDF4 instead + # parse with netcdftime instead raise OutOfBoundsDatetime assert dates.dtype == 'datetime64[ns]' @@ -293,7 +312,7 @@ def encode_cf_datetime(dates, units=None, calendar=None): num = (dates - ref_date) / time_delta except (OutOfBoundsDatetime, OverflowError): - num = _encode_datetime_with_netcdf4(dates, units, calendar) + num = _encode_datetime_with_netcdftime(dates, units, calendar) num = cast_to_int_if_safe(num) return (num, units, calendar) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index dadcdeff640..7c9528d741d 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -68,6 +68,7 @@ def _importorskip(modname, minversion=None): has_netCDF4, requires_netCDF4 = _importorskip('netCDF4') has_h5netcdf, requires_h5netcdf = _importorskip('h5netcdf') has_pynio, requires_pynio = _importorskip('Nio') +has_netcdftime, requires_netcdftime = _importorskip('netcdftime') has_dask, requires_dask = _importorskip('dask') has_bottleneck, requires_bottleneck = _importorskip('bottleneck') has_rasterio, requires_rasterio = _importorskip('rasterio') diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 6a509368017..4520e7aefef 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -13,8 +13,8 @@ from xarray.core import utils, indexing from xarray.testing import assert_identical from . import ( - TestCase, requires_netCDF4, unittest, raises_regex, IndexerMaker, - assert_array_equal) + TestCase, requires_netCDF4, requires_netcdftime, unittest, raises_regex, + IndexerMaker, assert_array_equal) from .test_backends import CFEncodedDataTest from xarray.core.pycompat import iteritems from xarray.backends.memory import InMemoryDataStore @@ -181,7 +181,7 @@ def test_decode_cf_with_conflicting_fill_missing_value(): assert_identical(actual, expected) -@requires_netCDF4 +@requires_netcdftime class TestEncodeCFVariable(TestCase): def test_incompatible_attributes(self): invalid_vars = [ @@ -237,7 +237,7 @@ def test_multidimensional_coordinates(self): assert 'coordinates' not in attrs -@requires_netCDF4 +@requires_netcdftime class TestDecodeCF(TestCase): def test_dataset(self): original = Dataset({ @@ -303,7 +303,7 @@ def test_invalid_time_units_raises_eagerly(self): with raises_regex(ValueError, 'unable to decode time'): decode_cf(ds) - @requires_netCDF4 + @requires_netcdftime def test_dataset_repr_with_netcdf4_datetimes(self): # regression test for #347 attrs = {'units': 'days since 0001-01-01', 'calendar': 'noleap'} @@ -316,7 +316,7 @@ def test_dataset_repr_with_netcdf4_datetimes(self): ds = decode_cf(Dataset({'time': ('time', [0, 1], attrs)})) assert '(time) datetime64[ns]' in repr(ds) - @requires_netCDF4 + @requires_netcdftime def test_decode_cf_datetime_transition_to_invalid(self): # manually create dataset with not-decoded date from datetime import datetime From 18a737b8ce7b57a903298b521ffacae116db11c5 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 20 Feb 2018 07:27:11 +0100 Subject: [PATCH 022/282] `axis` keyword ignored when applying `np.squeeze` to `DataArray` (#1487) (#1918) * `axis` keyword ignored when applying `np.squeeze` to `DataArray` (#1487) * Fixing style errors. --- doc/whats-new.rst | 2 ++ xarray/core/common.py | 22 ++++++++++++++++++---- xarray/tests/test_dataarray.py | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 42c0891f1a6..c54c67d1c4a 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -93,6 +93,8 @@ Enhancements - Speed of reindexing/alignment with dask array is orders of magnitude faster when inserting missing values (:issue:`1847`). By `Stephan Hoyer `_. +- Fix ``axis`` keyword ignored when applying ``np.squeeze`` to ``DataArray`` (:issue:`1487`). + By `Florian Pinault `_. - Add ``netcdftime`` as an optional dependency of xarray. This allows for encoding/decoding of datetimes with non-standard calendars without the netCDF4 dependency (:issue:`1084`). diff --git a/xarray/core/common.py b/xarray/core/common.py index 1366d0ff03d..094e8350eb6 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -211,14 +211,26 @@ def _ipython_key_completions_(self): return list(set(item_lists)) -def get_squeeze_dims(xarray_obj, dim): +def get_squeeze_dims(xarray_obj, dim, axis=None): """Get a list of dimensions to squeeze out. """ - if dim is None: + if not dim is None and not axis is None: + raise ValueError('cannot use both parameters `axis` and `dim`') + + if dim is None and axis is None: dim = [d for d, s in xarray_obj.sizes.items() if s == 1] else: if isinstance(dim, basestring): dim = [dim] + if isinstance(axis, int): + axis = (axis, ) + if isinstance(axis, tuple): + for a in axis: + if not isinstance(a, int): + raise ValueError( + 'parameter `axis` must be int or tuple of int.') + alldims = list(xarray_obj.sizes.keys()) + dim = [alldims[a] for a in axis] if any(xarray_obj.sizes[k] > 1 for k in dim): raise ValueError('cannot select a dimension to squeeze out ' 'which has length greater than one') @@ -228,7 +240,7 @@ def get_squeeze_dims(xarray_obj, dim): class BaseDataObject(AttrAccessMixin): """Shared base class for Dataset and DataArray.""" - def squeeze(self, dim=None, drop=False): + def squeeze(self, dim=None, drop=False, axis=None): """Return a new object with squeezed data. Parameters @@ -240,6 +252,8 @@ def squeeze(self, dim=None, drop=False): drop : bool, optional If ``drop=True``, drop squeezed coordinates instead of making them scalar. + axis : int, optional + Select the dimension to squeeze. Added for compatibility reasons. Returns ------- @@ -251,7 +265,7 @@ def squeeze(self, dim=None, drop=False): -------- numpy.squeeze """ - dims = get_squeeze_dims(self, dim) + dims = get_squeeze_dims(self, dim, axis) return self.isel(drop=drop, **{d: 0 for d in dims}) def get_index(self, key): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 0def5b6886e..cd8a209d5ac 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1668,6 +1668,21 @@ def test_squeeze_drop(self): actual = array.squeeze(drop=False) assert_identical(expected, actual) + array = DataArray([[[0., 1.]]], dims=['dim_0', 'dim_1', 'dim_2']) + expected = DataArray([[0., 1.]], dims=['dim_1', 'dim_2']) + actual = array.squeeze(axis=0) + assert_identical(expected, actual) + + array = DataArray([[[[0., 1.]]]], dims=[ + 'dim_0', 'dim_1', 'dim_2', 'dim_3']) + expected = DataArray([[0., 1.]], dims=['dim_1', 'dim_3']) + actual = array.squeeze(axis=(0, 2)) + assert_identical(expected, actual) + + array = DataArray([[[0., 1.]]], dims=['dim_0', 'dim_1', 'dim_2']) + with pytest.raises(ValueError): + array.squeeze(axis=0, dim='dim_1') + def test_drop_coordinates(self): expected = DataArray(np.random.randn(2, 3), dims=['x', 'y']) arr = expected.copy() From 97f5778261e48391ba6772ca518cd2a51ff0ec83 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Tue, 20 Feb 2018 13:04:58 -0500 Subject: [PATCH 023/282] flake8 passes (#1925) * flake8 passes * Fixing style errors. * pray to @stickler-ci * update stickler config * gitignore --- .gitignore | 3 +++ .stickler.yml | 3 +++ asv_bench/benchmarks/__init__.py | 3 +-- asv_bench/benchmarks/dataarray_missing.py | 4 ++-- asv_bench/benchmarks/indexing.py | 2 +- doc/examples/_code/accessor_example.py | 1 + doc/examples/_code/weather_data_setup.py | 1 - doc/gallery/plot_cartopy_facetgrid.py | 6 ++++-- doc/gallery/plot_colorbar_center.py | 10 +++++----- setup.cfg | 3 +++ setup.py | 17 +++++++++++------ xarray/backends/rasterio_.py | 4 ++-- xarray/backends/zarr.py | 4 ++-- xarray/core/dataset.py | 6 ++++-- xarray/core/indexing.py | 2 +- xarray/core/pycompat.py | 10 ++++++---- xarray/plot/plot.py | 4 ++-- xarray/testing.py | 6 +++--- xarray/tests/test_backends.py | 17 ++++++++++------- xarray/tests/test_dataset.py | 2 +- xarray/tests/test_duck_array_ops.py | 21 +++++++++++---------- xarray/tests/test_indexing.py | 10 +++++----- xarray/tests/test_ufuncs.py | 2 +- 23 files changed, 82 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index 490eb49f9d4..b573471940d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,9 @@ pip-log.txt nosetests.xml .cache .ropeproject/ +.tags* +.testmondata +.pytest_cache # asv environments .asv diff --git a/.stickler.yml b/.stickler.yml index 1634c1f53c8..fc165161fd4 100644 --- a/.stickler.yml +++ b/.stickler.yml @@ -2,6 +2,9 @@ linters: flake8: max-line-length: 79 fixer: true + ignore: I002 + exclude: + - 'doc/' py3k: fixers: enable: true diff --git a/asv_bench/benchmarks/__init__.py b/asv_bench/benchmarks/__init__.py index e2f49e6ab48..f9bbc751284 100644 --- a/asv_bench/benchmarks/__init__.py +++ b/asv_bench/benchmarks/__init__.py @@ -2,7 +2,6 @@ from __future__ import division from __future__ import print_function import itertools -import random import numpy as np @@ -11,7 +10,7 @@ def requires_dask(): try: - import dask + import dask # noqa except ImportError: raise NotImplementedError diff --git a/asv_bench/benchmarks/dataarray_missing.py b/asv_bench/benchmarks/dataarray_missing.py index c6aa8f428bd..e0127bb7da2 100644 --- a/asv_bench/benchmarks/dataarray_missing.py +++ b/asv_bench/benchmarks/dataarray_missing.py @@ -5,7 +5,7 @@ import pandas as pd try: - import dask + import dask # noqa except ImportError: pass @@ -17,7 +17,7 @@ def make_bench_data(shape, frac_nan, chunks): vals = randn(shape, frac_nan) coords = {'time': pd.date_range('2000-01-01', freq='D', - periods=shape[0])} + periods=shape[0])} da = xr.DataArray(vals, dims=('time', 'x', 'y'), coords=coords) if chunks is not None: diff --git a/asv_bench/benchmarks/indexing.py b/asv_bench/benchmarks/indexing.py index e9a85115a49..9a41c6cf0e7 100644 --- a/asv_bench/benchmarks/indexing.py +++ b/asv_bench/benchmarks/indexing.py @@ -29,7 +29,7 @@ outer_indexes = { '1d': {'x': randint(0, nx, 400)}, - '2d': {'x': randint(0, nx, 500), 'y': randint(0, ny, 400)}, + '2d': {'x': randint(0, nx, 500), 'y': randint(0, ny, 400)}, '2d-1scalar': {'x': randint(0, nx, 100), 'y': 1, 't': randint(0, nt, 400)} } diff --git a/doc/examples/_code/accessor_example.py b/doc/examples/_code/accessor_example.py index 1c846b38687..a11ebf9329b 100644 --- a/doc/examples/_code/accessor_example.py +++ b/doc/examples/_code/accessor_example.py @@ -1,5 +1,6 @@ import xarray as xr + @xr.register_dataset_accessor('geo') class GeoAccessor(object): def __init__(self, xarray_obj): diff --git a/doc/examples/_code/weather_data_setup.py b/doc/examples/_code/weather_data_setup.py index a6190ad3cfe..b370aea00f2 100644 --- a/doc/examples/_code/weather_data_setup.py +++ b/doc/examples/_code/weather_data_setup.py @@ -1,7 +1,6 @@ import xarray as xr import numpy as np import pandas as pd -import seaborn as sns # pandas aware plotting library np.random.seed(123) diff --git a/doc/gallery/plot_cartopy_facetgrid.py b/doc/gallery/plot_cartopy_facetgrid.py index 525ae7054b0..10d782ca41f 100644 --- a/doc/gallery/plot_cartopy_facetgrid.py +++ b/doc/gallery/plot_cartopy_facetgrid.py @@ -12,7 +12,9 @@ For more details see `this discussion`_ on github. .. _this discussion: https://github.com/pydata/xarray/issues/1397#issuecomment-299190567 -""" +""" # noqa + +from __future__ import division import xarray as xr import cartopy.crs as ccrs @@ -27,7 +29,7 @@ p = air.plot(transform=ccrs.PlateCarree(), # the data's projection col='time', col_wrap=1, # multiplot settings - aspect=ds.dims['lon']/ds.dims['lat'], # for a sensible figsize + aspect=ds.dims['lon'] / ds.dims['lat'], # for a sensible figsize subplot_kws={'projection': map_proj}) # the plot's projection # We have to set the map's options on all four axes diff --git a/doc/gallery/plot_colorbar_center.py b/doc/gallery/plot_colorbar_center.py index 00c25af50d4..ce83b4f05cf 100644 --- a/doc/gallery/plot_colorbar_center.py +++ b/doc/gallery/plot_colorbar_center.py @@ -15,26 +15,26 @@ ds = xr.tutorial.load_dataset('air_temperature') air = ds.air.isel(time=0) -f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(8, 6)) +f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(8, 6)) # The first plot (in kelvins) chooses "viridis" and uses the data's min/max -air.plot(ax=ax1, cbar_kwargs={'label':'K'}) +air.plot(ax=ax1, cbar_kwargs={'label': 'K'}) ax1.set_title('Kelvins: default') ax2.set_xlabel('') # The second plot (in celsius) now chooses "BuRd" and centers min/max around 0 airc = air - 273.15 -airc.plot(ax=ax2, cbar_kwargs={'label':'°C'}) +airc.plot(ax=ax2, cbar_kwargs={'label': '°C'}) ax2.set_title('Celsius: default') ax2.set_xlabel('') ax2.set_ylabel('') # The center doesn't have to be 0 -air.plot(ax=ax3, center=273.15, cbar_kwargs={'label':'K'}) +air.plot(ax=ax3, center=273.15, cbar_kwargs={'label': 'K'}) ax3.set_title('Kelvins: center=273.15') # Or it can be ignored -airc.plot(ax=ax4, center=False, cbar_kwargs={'label':'°C'}) +airc.plot(ax=ax4, center=False, cbar_kwargs={'label': '°C'}) ax4.set_title('Celsius: center=False') ax4.set_ylabel('') diff --git a/setup.cfg b/setup.cfg index d2f336aa1d0..c7df2b7fc74 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,3 +6,6 @@ python_files=test_*.py [flake8] max-line-length=79 +ignore=I002 +exclude= + doc/ diff --git a/setup.py b/setup.py index ccffc6369e8..ee717950dbe 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ import warnings from setuptools import setup, find_packages -from setuptools import Command MAJOR = 0 MINOR = 10 @@ -64,7 +63,7 @@ - Issue tracker: http://github.com/pydata/xarray/issues - Source code: http://github.com/pydata/xarray - SciPy2015 talk: https://www.youtube.com/watch?v=X0pAhJgySxk -""" +""" # noqa # Code to extract and write the version copied from pandas. # Used under the terms of pandas's license, see licenses/PANDAS_LICENSE. @@ -84,18 +83,23 @@ (so, serr) = pipe.communicate() if pipe.returncode == 0: break - except: + except BaseException: pass if pipe is None or pipe.returncode != 0: # no git, or not in git dir if os.path.exists('xarray/version.py'): - warnings.warn("WARNING: Couldn't get git revision, using existing xarray/version.py") + warnings.warn( + "WARNING: Couldn't get git revision," + " using existing xarray/version.py") write_version = False else: - warnings.warn("WARNING: Couldn't get git revision, using generic version string") + warnings.warn( + "WARNING: Couldn't get git revision," + " using generic version string") else: - # have git, in git dir, but may have used a shallow clone (travis does this) + # have git, in git dir, but may have used a shallow clone (travis does + # this) rev = so.strip() # makes distutils blow up on Python 2.7 if sys.version_info[0] >= 3: @@ -132,6 +136,7 @@ def write_version_py(filename=None): finally: a.close() + if write_version: write_version_py() diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index c624c1f5ff8..b3b94c86c3c 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -190,8 +190,8 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, if parse: nx, ny = riods.width, riods.height # xarray coordinates are pixel centered - x, _ = (np.arange(nx)+0.5, np.zeros(nx)+0.5) * transform - _, y = (np.zeros(ny)+0.5, np.arange(ny)+0.5) * transform + x, _ = (np.arange(nx) + 0.5, np.zeros(nx) + 0.5) * transform + _, y = (np.zeros(ny) + 0.5, np.arange(ny) + 0.5) * transform coords['y'] = y coords['x'] = x else: diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 2737d9fb213..12149026ac3 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -29,7 +29,7 @@ def _encode_zarr_attr_value(value): encoded = value.item() # np.string_('X').item() returns a type `bytes` # zarr still doesn't like that - if type(encoded) is bytes: + if type(encoded) is bytes: # noqa encoded = b64encode(encoded) else: encoded = value @@ -37,7 +37,7 @@ def _encode_zarr_attr_value(value): def _ensure_valid_fill_value(value, dtype): - if dtype.type == np.string_ and type(value) == bytes: + if dtype.type == np.string_ and type(value) == bytes: # noqa valid = b64encode(value) else: valid = value diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 5f31a3d9483..2e5c9a84b31 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1516,7 +1516,8 @@ def take(variable, slices): # Note: remove helper function when once when numpy # supports vindex https://github.com/numpy/numpy/pull/6075 if hasattr(variable.data, 'vindex'): - # Special case for dask backed arrays to use vectorised list indexing + # Special case for dask backed arrays to use vectorised list + # indexing sel = variable.data.vindex[slices] else: # Otherwise assume backend is numpy array with 'fancy' indexing @@ -1579,7 +1580,8 @@ def relevant_keys(mapping): variables = OrderedDict() for name, var in reordered.variables.items(): - if name in indexers_dict or any(d in indexer_dims for d in var.dims): + if name in indexers_dict or any( + d in indexer_dims for d in var.dims): # slice if var is an indexer or depends on an indexed dim slc = [indexers_dict[k] if k in indexers_dict diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index e06b045ad88..e902aa8ffa3 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -292,7 +292,7 @@ class ExplicitIndexer(object): """ def __init__(self, key): - if type(self) is ExplicitIndexer: + if type(self) is ExplicitIndexer: # noqa raise TypeError('cannot instantiate base ExplicitIndexer objects') self._key = tuple(key) diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 4b83df9e14f..19c16e445a6 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -108,7 +108,8 @@ def __exit__(self, exctype, excinst, exctb): # exactly reproduce the limitations of the CPython interpreter. # # See http://bugs.python.org/issue12029 for more details - return exctype is not None and issubclass(exctype, self._exceptions) + return exctype is not None and issubclass( + exctype, self._exceptions) try: from contextlib import ExitStack except ImportError: @@ -185,7 +186,8 @@ def enter_context(self, cm): If successful, also pushes its __exit__ method as a callback and returns the result of the __enter__ method. """ - # We look up the special methods on the type to match the with statement + # We look up the special methods on the type to match the with + # statement _cm_type = type(cm) _exit = _cm_type.__exit__ result = _cm_type.__enter__(cm) @@ -208,7 +210,7 @@ def __exit__(self, *exc_details): def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain - while 1: + while True: exc_context = new_exc.__context__ if exc_context is old_exc: # Context is already set correctly (see issue 20317) @@ -231,7 +233,7 @@ def _fix_exception_context(new_exc, old_exc): suppressed_exc = True pending_raise = False exc_details = (None, None, None) - except: + except BaseException: new_exc_details = sys.exc_info() # simulate the stack of exceptions by setting the context _fix_exception_context(new_exc_details[1], exc_details[1]) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 162a0c238f5..57e3f101f4f 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -759,9 +759,9 @@ def _is_monotonic(coord, axis=0): else: n = coord.shape[axis] delta_pos = (coord.take(np.arange(1, n), axis=axis) >= - coord.take(np.arange(0, n-1), axis=axis)) + coord.take(np.arange(0, n - 1), axis=axis)) delta_neg = (coord.take(np.arange(1, n), axis=axis) <= - coord.take(np.arange(0, n-1), axis=axis)) + coord.take(np.arange(0, n - 1), axis=axis)) return np.all(delta_pos) or np.all(delta_neg) diff --git a/xarray/testing.py b/xarray/testing.py index f51e474405f..6b0a5b736de 100644 --- a/xarray/testing.py +++ b/xarray/testing.py @@ -50,7 +50,7 @@ def assert_equal(a, b): """ import xarray as xr __tracebackhide__ = True # noqa: F841 - assert type(a) == type(b) + assert type(a) == type(b) # noqa if isinstance(a, (xr.Variable, xr.DataArray, xr.Dataset)): assert a.equals(b), '{}\n{}'.format(a, b) else: @@ -77,7 +77,7 @@ def assert_identical(a, b): """ import xarray as xr __tracebackhide__ = True # noqa: F841 - assert type(a) == type(b) + assert type(a) == type(b) # noqa if isinstance(a, xr.DataArray): assert a.name == b.name assert_identical(a._to_temp_dataset(), b._to_temp_dataset()) @@ -115,7 +115,7 @@ def assert_allclose(a, b, rtol=1e-05, atol=1e-08, decode_bytes=True): """ import xarray as xr __tracebackhide__ = True # noqa: F841 - assert type(a) == type(b) + assert type(a) == type(b) # noqa kwargs = dict(rtol=rtol, atol=atol, decode_bytes=decode_bytes) if isinstance(a, xr.Variable): assert a.dims == b.dims diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index e88fc790571..f82196212b0 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -630,7 +630,7 @@ def test_roundtrip_endian(self): # should still pass though. assert_identical(ds, actual) - if type(self) is NetCDF4DataTest: + if isinstance(self, NetCDF4DataTest): ds['z'].encoding['endian'] = 'big' with pytest.raises(NotImplementedError): with self.roundtrip(ds) as actual: @@ -2214,7 +2214,10 @@ def create_tmp_geotiff(nx=4, ny=3, nz=3, else: data_shape = nz, ny, nx write_kwargs = {} - data = np.arange(nz*ny*nx, dtype=rasterio.float32).reshape(*data_shape) + data = np.arange( + nz * ny * nx, + dtype=rasterio.float32).reshape( + *data_shape) if transform is None: transform = from_origin(*transform_args) with rasterio.open( @@ -2231,10 +2234,10 @@ def create_tmp_geotiff(nx=4, ny=3, nz=3, data = data[np.newaxis, ...] if nz == 1 else data expected = DataArray(data, dims=('band', 'y', 'x'), coords={ - 'band': np.arange(nz)+1, - 'y': -np.arange(ny) * d + b + dy/2, - 'x': np.arange(nx) * c + a + dx/2, - }) + 'band': np.arange(nz) + 1, + 'y': -np.arange(ny) * d + b + dy / 2, + 'x': np.arange(nx) * c + a + dx / 2, + }) yield tmp_file, expected @@ -2316,7 +2319,7 @@ def test_notransform(self): with create_tmp_file(suffix='.tif') as tmp_file: # data nx, ny, nz = 4, 3, 3 - data = np.arange(nx*ny*nz, + data = np.arange(nx * ny * nz, dtype=rasterio.float32).reshape(nz, ny, nx) with rasterio.open( tmp_file, 'w', diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 09d67613007..4e746b90635 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -407,7 +407,7 @@ def test_properties(self): # change them inadvertently: assert isinstance(ds.dims, utils.Frozen) assert isinstance(ds.dims.mapping, utils.SortedKeysDict) - assert type(ds.dims.mapping.mapping) is dict + assert type(ds.dims.mapping.mapping) is dict # noqa with pytest.warns(FutureWarning): self.assertItemsEqual(ds, list(ds.variables)) diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index d68a7a382de..e7dc137e72a 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -17,14 +17,15 @@ class TestOps(TestCase): + def setUp(self): - self.x = array([[[nan, nan, 2., nan], - [nan, 5., 6., nan], - [8., 9., 10., nan]], + self.x = array([[[nan, nan, 2., nan], + [nan, 5., 6., nan], + [8., 9., 10., nan]], - [[nan, 13., 14., 15.], - [nan, 17., 18., nan], - [nan, 21., nan, nan]]]) + [[nan, 13., 14., 15.], + [nan, 17., 18., nan], + [nan, 21., nan, nan]]]) def test_first(self): expected_results = [array([[nan, 13, 2, 15], @@ -287,12 +288,12 @@ def test_argmin_max(dim_num, dtype, contains_nan, dask, func, skipna, aggdim): if aggdim == 'y' and contains_nan and skipna: with pytest.raises(ValueError): actual = da.isel(**{ - aggdim: getattr(da, 'arg'+func)(dim=aggdim, - skipna=skipna).compute()}) + aggdim: getattr(da, 'arg' + func)(dim=aggdim, + skipna=skipna).compute()}) return - actual = da.isel(**{ - aggdim: getattr(da, 'arg'+func)(dim=aggdim, skipna=skipna).compute()}) + actual = da.isel(**{aggdim: getattr(da, 'arg' + func) + (dim=aggdim, skipna=skipna).compute()}) expected = getattr(da, func)(dim=aggdim, skipna=skipna) assert_allclose(actual.drop(actual.coords), expected.drop(expected.coords)) diff --git a/xarray/tests/test_indexing.py b/xarray/tests/test_indexing.py index 3d93afb26d4..4729aad9b79 100644 --- a/xarray/tests/test_indexing.py +++ b/xarray/tests/test_indexing.py @@ -114,25 +114,25 @@ def test_indexer(data, x, expected_pos, expected_idx=None): test_indexer(data, Variable([], 1), 0) test_indexer(mdata, ('a', 1, -1), 0) test_indexer(mdata, ('a', 1), - [True, True, False, False, False, False, False, False], + [True, True, False, False, False, False, False, False], [-1, -2]) test_indexer(mdata, 'a', slice(0, 4, None), pd.MultiIndex.from_product([[1, 2], [-1, -2]])) test_indexer(mdata, ('a',), - [True, True, True, True, False, False, False, False], + [True, True, True, True, False, False, False, False], pd.MultiIndex.from_product([[1, 2], [-1, -2]])) test_indexer(mdata, [('a', 1, -1), ('b', 2, -2)], [0, 7]) test_indexer(mdata, slice('a', 'b'), slice(0, 8, None)) test_indexer(mdata, slice(('a', 1), ('b', 1)), slice(0, 6, None)) test_indexer(mdata, {'one': 'a', 'two': 1, 'three': -1}, 0) test_indexer(mdata, {'one': 'a', 'two': 1}, - [True, True, False, False, False, False, False, False], + [True, True, False, False, False, False, False, False], [-1, -2]) test_indexer(mdata, {'one': 'a', 'three': -1}, - [True, False, True, False, False, False, False, False], + [True, False, True, False, False, False, False, False], [1, 2]) test_indexer(mdata, {'one': 'a'}, - [True, True, True, True, False, False, False, False], + [True, True, True, True, False, False, False, False], pd.MultiIndex.from_product([[1, 2], [-1, -2]])) diff --git a/xarray/tests/test_ufuncs.py b/xarray/tests/test_ufuncs.py index a42819605fa..0d56285dfc1 100644 --- a/xarray/tests/test_ufuncs.py +++ b/xarray/tests/test_ufuncs.py @@ -14,7 +14,7 @@ class TestOps(TestCase): def assert_identical(self, a, b): - assert type(a) is type(b) or (float(a) == float(b)) + assert type(a) is type(b) or (float(a) == float(b)) # noqa try: assert a.identical(b), (a, b) except AttributeError: From 697cc74b9af5fbfedadd54fd07019ce7684553ec Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Wed, 21 Feb 2018 01:18:34 -0500 Subject: [PATCH 024/282] Use requires_netcdftime decorators in test_coding_times.py (#1929) * Switch requires_netCDF4 decorators to requires_netcdftime decorators in test_coding_times.py * Fix 'netcdftime' has no attribute 'netcdftime' error --- xarray/tests/test_coding_times.py | 60 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index eb5c03b1f95..092559ce9da 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -8,8 +8,9 @@ import pandas as pd from xarray import Variable, coding +from xarray.coding.times import _import_netcdftime from . import ( - TestCase, requires_netCDF4, assert_array_equal) + TestCase, requires_netcdftime, assert_array_equal) import pytest @@ -22,9 +23,9 @@ def _ensure_naive_tz(dt): class TestDatetime(TestCase): - @requires_netCDF4 + @requires_netcdftime def test_cf_datetime(self): - import netCDF4 as nc4 + nctime = _import_netcdftime() for num_dates, units in [ (np.arange(10), 'days since 2000-01-01'), (np.arange(10).astype('float64'), 'days since 2000-01-01'), @@ -53,7 +54,7 @@ def test_cf_datetime(self): ]: for calendar in ['standard', 'gregorian', 'proleptic_gregorian']: expected = _ensure_naive_tz( - nc4.num2date(num_dates, units, calendar)) + nctime.num2date(num_dates, units, calendar)) print(num_dates, units, calendar) with warnings.catch_warnings(): warnings.filterwarnings('ignore', @@ -88,7 +89,7 @@ def test_cf_datetime(self): pd.Index(actual), units, calendar) assert_array_equal(num_dates, np.around(encoded, 1)) - @requires_netCDF4 + @requires_netcdftime def test_decode_cf_datetime_overflow(self): # checks for # https://github.com/pydata/pandas/issues/14068 @@ -113,7 +114,7 @@ def test_decode_cf_datetime_non_standard_units(self): actual = coding.times.decode_cf_datetime(np.arange(100), units) assert_array_equal(actual, expected) - @requires_netCDF4 + @requires_netcdftime def test_decode_cf_datetime_non_iso_strings(self): # datetime strings that are _almost_ ISO compliant but not quite, # but which netCDF4.num2date can still parse correctly @@ -125,17 +126,17 @@ def test_decode_cf_datetime_non_iso_strings(self): actual = coding.times.decode_cf_datetime(num_dates, units) assert_array_equal(actual, expected) - @requires_netCDF4 + @requires_netcdftime def test_decode_non_standard_calendar(self): - import netCDF4 as nc4 + nctime = _import_netcdftime() for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', '366_day']: units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') - noleap_time = nc4.date2num(times.to_pydatetime(), units, - calendar=calendar) + noleap_time = nctime.date2num(times.to_pydatetime(), units, + calendar=calendar) expected = times.values with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') @@ -148,7 +149,7 @@ def test_decode_non_standard_calendar(self): # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, 's')).all() - @requires_netCDF4 + @requires_netcdftime def test_decode_non_standard_calendar_single_element(self): units = 'days since 0001-01-01' for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', @@ -161,34 +162,37 @@ def test_decode_non_standard_calendar_single_element(self): calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') - @requires_netCDF4 + @requires_netcdftime def test_decode_non_standard_calendar_single_element_fallback(self): - import netCDF4 as nc4 + nctime = _import_netcdftime() units = 'days since 0001-01-01' - dt = nc4.netcdftime.datetime(2001, 2, 29) + try: + dt = nctime.netcdftime.datetime(2001, 2, 29) + except AttributeError: + # Must be using standalone netcdftime library + dt = nctime.datetime(2001, 2, 29) for calendar in ['360_day', 'all_leap', '366_day']: - num_time = nc4.date2num(dt, units, calendar) + num_time = nctime.date2num(dt, units, calendar) with pytest.warns(Warning, match='Unable to decode time axis'): actual = coding.times.decode_cf_datetime(num_time, units, calendar=calendar) - expected = np.asarray(nc4.num2date(num_time, units, calendar)) - print(num_time, calendar, actual, expected) + expected = np.asarray(nctime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual - @requires_netCDF4 + @requires_netcdftime def test_decode_non_standard_calendar_multidim_time(self): - import netCDF4 as nc4 + nctime = _import_netcdftime() calendar = 'noleap' units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = nc4.date2num(times1.to_pydatetime(), units, - calendar=calendar) - noleap_time2 = nc4.date2num(times2.to_pydatetime(), units, - calendar=calendar) + noleap_time1 = nctime.date2num(times1.to_pydatetime(), units, + calendar=calendar) + noleap_time2 = nctime.date2num(times2.to_pydatetime(), units, + calendar=calendar) mdim_time = np.empty((len(noleap_time1), 2), ) mdim_time[:, 0] = noleap_time1 mdim_time[:, 1] = noleap_time2 @@ -203,16 +207,16 @@ def test_decode_non_standard_calendar_multidim_time(self): assert_array_equal(actual[:, 0], expected1) assert_array_equal(actual[:, 1], expected2) - @requires_netCDF4 + @requires_netcdftime def test_decode_non_standard_calendar_fallback(self): - import netCDF4 as nc4 + nctime = _import_netcdftime() # ensure leap year doesn't matter for year in [2010, 2011, 2012, 2013, 2014]: for calendar in ['360_day', '366_day', 'all_leap']: calendar = '360_day' units = 'days since {0}-01-01'.format(year) num_times = np.arange(100) - expected = nc4.num2date(num_times, units, calendar) + expected = nctime.num2date(num_times, units, calendar) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') @@ -225,7 +229,7 @@ def test_decode_non_standard_calendar_fallback(self): assert actual.dtype == np.dtype('O') assert_array_equal(actual, expected) - @requires_netCDF4 + @requires_netcdftime def test_cf_datetime_nan(self): for num_dates, units, expected_list in [ ([np.nan], 'days since 2000-01-01', ['NaT']), @@ -240,7 +244,7 @@ def test_cf_datetime_nan(self): expected = np.array(expected_list, dtype='datetime64[ns]') assert_array_equal(expected, actual) - @requires_netCDF4 + @requires_netcdftime def test_decoded_cf_datetime_array_2d(self): # regression test for GH1229 variable = Variable(('x', 'y'), np.array([[0, 1], [2, 3]]), From fc7fe4875289778014fc1ea04b6b09be12f9750a Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Fri, 23 Feb 2018 14:55:46 -0500 Subject: [PATCH 025/282] DOC: Add contributing section in README.rst and README.rst in doc/ (#1934) * Add Contributing section * Add install sphinx modules before building the docs * use double quotes * typo * Create README.rst * improve reading text * add url from docs online * added url from online docs --- README.rst | 5 +++++ doc/README.rst | 4 ++++ doc/contributing.rst | 7 +++++-- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 doc/README.rst diff --git a/README.rst b/README.rst index 8e77f55ccbb..37061e2c6ce 100644 --- a/README.rst +++ b/README.rst @@ -84,6 +84,11 @@ Documentation The official documentation is hosted on ReadTheDocs at http://xarray.pydata.org/ +Contributing +------------ + +You can find information about contributing to xarray at our `Contributing page `_. + Get in touch ------------ diff --git a/doc/README.rst b/doc/README.rst new file mode 100644 index 00000000000..af7bc96092c --- /dev/null +++ b/doc/README.rst @@ -0,0 +1,4 @@ +xarray +------ + +You can find information about building the docs at our `Contributing page `_. diff --git a/doc/contributing.rst b/doc/contributing.rst index c389321764f..89f7eeeec59 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -290,8 +290,11 @@ First, you need to have a development environment to be able to build xarray Building the documentation ~~~~~~~~~~~~~~~~~~~~~~~~~~ -So how do you build the docs? Navigate to your local -``xarray/doc/`` directory in the console and run:: +In your development environment, install ``sphinx`` and ``sphinx_rtd_theme``:: + + conda install -c anaconda sphinx sphinx_rtd_theme + +Navigate to your local ``xarray/doc/`` directory in the console and run:: make html From f0535679a79a7f8276b276a92c3d9984504adde0 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 25 Feb 2018 12:49:15 -0800 Subject: [PATCH 026/282] Tweak stickler config to ignore Python files in the docs. (#1936) Tweak stickler config: ignore Python files in the docs & disable fixer --- .stickler.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.stickler.yml b/.stickler.yml index fc165161fd4..d708ec10ec1 100644 --- a/.stickler.yml +++ b/.stickler.yml @@ -1,10 +1,8 @@ linters: flake8: max-line-length: 79 - fixer: true + fixer: false ignore: I002 exclude: - - 'doc/' + - doc/* py3k: -fixers: - enable: true From ec6e1603680ffa5a975baf890953cf2605394f0c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 25 Feb 2018 12:52:16 -0800 Subject: [PATCH 027/282] Fix/dask isnull (#1939) * fix isnull for dask arrays and add test to test_duck_array_ops.py * minor cleanup / proper test skipping in test_duck_array_ops.py * whatsnew * Fixing style errors. * flake8 --- doc/whats-new.rst | 3 +++ xarray/core/ops.py | 4 ++-- xarray/tests/test_duck_array_ops.py | 34 +++++++++++++++-------------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index c54c67d1c4a..fa4bfb7d4e8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -167,6 +167,9 @@ Bug fixes match pandas (:issue:`1847`). A combination of strings/numbers or unicode/bytes now promote to object dtype, instead of strings or unicode. By `Stephan Hoyer `_. + - Fixed bug where :py:meth:`~xarray.DataArray.isnull` was loading data + stored as dask arrays (:issue:`1937`). + By `Joe Hamman `_. .. _whats-new.0.10.0: diff --git a/xarray/core/ops.py b/xarray/core/ops.py index d02b8fa3108..f9e1e3ba355 100644 --- a/xarray/core/ops.py +++ b/xarray/core/ops.py @@ -12,7 +12,6 @@ import operator import numpy as np -import pandas as pd from . import dtypes from . import duck_array_ops @@ -320,7 +319,8 @@ def inject_all_ops_and_reduce_methods(cls, priority=50, array_only=True): setattr(cls, name, cls._unary_op(_method_wrapper(name))) for name in PANDAS_UNARY_FUNCTIONS: - f = _func_slash_method_wrapper(getattr(pd, name), name=name) + f = _func_slash_method_wrapper( + getattr(duck_array_ops, name), name=name) setattr(cls, name, cls._unary_op(f)) f = _func_slash_method_wrapper(duck_array_ops.around, name='round') diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index e7dc137e72a..5bb7e09d918 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -9,11 +9,12 @@ from xarray.core.duck_array_ops import ( first, last, count, mean, array_notnull_equiv, where, stack, concatenate ) +from xarray.core.pycompat import dask_array_type from xarray import DataArray -from xarray.testing import assert_allclose +from xarray.testing import assert_allclose, assert_equal from xarray import concat -from . import TestCase, raises_regex, has_dask +from . import TestCase, raises_regex, has_dask, requires_dask class TestOps(TestCase): @@ -194,23 +195,19 @@ def series_reduce(da, func, dim, **kwargs): def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): if aggdim == 'y' and dim_num < 2: - return + pytest.skip('dim not in this test') if dtype == np.bool_ and func == 'mean': - return # numpy does not support this + pytest.skip('numpy does not support this') if dask and not has_dask: - return + pytest.skip('requires dask') rtol = 1e-04 if dtype == np.float32 else 1e-05 da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask) axis = None if aggdim is None else da.get_axis_num(aggdim) - if dask and not skipna and func in ['var', 'std'] and dtype == np.bool_: - # TODO this might be dask's bug - return - if (LooseVersion(np.__version__) >= LooseVersion('1.13.0') and da.dtype.kind == 'O' and skipna): # Numpy < 1.13 does not handle object-type array. @@ -269,19 +266,17 @@ def test_argmin_max(dim_num, dtype, contains_nan, dask, func, skipna, aggdim): # just make sure da[da.argmin()] == da.min() if aggdim == 'y' and dim_num < 2: - return + pytest.skip('dim not in this test') if dask and not has_dask: - return + pytest.skip('requires dask') if contains_nan: if not skipna: - # numpy's argmin (not nanargmin) does not handle object-dtype - return + pytest.skip("numpy's argmin (not nanargmin) does not handle " + "object-dtype") if skipna and np.dtype(dtype).kind in 'iufc': - # numpy's nanargmin raises ValueError for all nan axis - return - + pytest.skip("numpy's nanargmin raises ValueError for all nan axis") da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, dask=dask) @@ -302,3 +297,10 @@ def test_argmin_max_error(): da = construct_dataarray(2, np.bool_, contains_nan=True, dask=False) with pytest.raises(ValueError): da.argmin(dim='y') + + +@requires_dask +def test_isnull_with_dask(): + da = construct_dataarray(2, np.float32, contains_nan=True, dask=True) + assert isinstance(da.isnull().data, dask_array_type) + assert_equal(da.isnull().load(), da.load().isnull()) From d463b9b46956f8796b26a4ace6572271306c8361 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 25 Feb 2018 16:20:24 -0800 Subject: [PATCH 028/282] Doc fixes/cleanup in anticipation of 0.10.1 release (#1940) --- doc/conf.py | 16 ++++++---- doc/contributing.rst | 5 +-- doc/index.rst | 2 +- doc/whats-new.rst | 75 +++++++++++++++++++++++++------------------- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index f92946ccc05..f4c7d7058e5 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -20,7 +20,7 @@ import datetime import importlib -allowed_failures = [] +allowed_failures = set() print("python exec:", sys.executable) print("sys.path:", sys.path) @@ -34,11 +34,15 @@ print("%s: %s, %s" % (name, module.__version__, fname)) except ImportError: print("no %s" % name) + # neither rasterio nor cartopy should be hard requirements for + # the doc build. if name == 'rasterio': - # not having rasterio should not break the build process - allowed_failures = ['gallery/plot_rasterio_rgb.py', - 'gallery/plot_rasterio.py' - ] + allowed_failures.update(['gallery/plot_rasterio_rgb.py', + 'gallery/plot_rasterio.py']) + elif name == 'cartopy': + allowed_failures.update(['gallery/plot_cartopy_facetgrid.py', + 'gallery/plot_rasterio_rgb.py', + 'gallery/plot_rasterio.py']) import xarray print("xarray: %s, %s" % (xarray.__version__, xarray.__file__)) @@ -70,7 +74,7 @@ sphinx_gallery_conf = {'examples_dirs': 'gallery', 'gallery_dirs': 'auto_gallery', 'backreferences_dir': False, - 'expected_failing_examples': allowed_failures + 'expected_failing_examples': list(allowed_failures) } autosummary_generate = True diff --git a/doc/contributing.rst b/doc/contributing.rst index 89f7eeeec59..26734e66ae6 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -290,9 +290,10 @@ First, you need to have a development environment to be able to build xarray Building the documentation ~~~~~~~~~~~~~~~~~~~~~~~~~~ -In your development environment, install ``sphinx`` and ``sphinx_rtd_theme``:: +In your development environment, install ``sphinx``, ``sphinx_rtd_theme``, +``sphinx-gallery`` and ``numpydoc``:: - conda install -c anaconda sphinx sphinx_rtd_theme + conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc Navigate to your local ``xarray/doc/`` directory in the console and run:: diff --git a/doc/index.rst b/doc/index.rst index 8389d598bcf..e9df085169a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -35,6 +35,7 @@ Documentation .. toctree:: :maxdepth: 1 + whats-new why-xarray faq examples @@ -53,7 +54,6 @@ Documentation api internals contributing - whats-new See also -------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fa4bfb7d4e8..7229c95490f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -30,19 +30,56 @@ What's New v0.10.1 (unreleased) -------------------- +The minor release includes a number of bug-fixes and backwards compatible enhancements. + Documentation ~~~~~~~~~~~~~ -- Added apply_ufunc example to toy weather data page (:issue:`1844`). +- Added a new guide on :ref:`contributing` (:issue:`640`) + By `Joe Hamman `_. +- Added apply_ufunc example to :ref:`toy weather data` (:issue:`1844`). By `Liam Brannigan `_. - New entry `Why don’t aggregations return Python scalars?` in the :doc:`faq` (:issue:`1726`). By `0x0L `_. -- Added a new contributors guide (:issue:`640`) - By `Joe Hamman `_. Enhancements ~~~~~~~~~~~~ + +**New functions and methods**: + +- Added :py:meth:`DataArray.to_iris` and + :py:meth:`DataArray.from_iris` for + converting data arrays to and from Iris_ Cubes with the same data and coordinates + (:issue:`621` and :issue:`37`). + By `Neil Parley `_ and `Duncan Watson-Parris `_. +- Experimental support for using `Zarr`_ as storage layer for xarray + (:issue:`1223`). + By `Ryan Abernathey `_ and + `Joe Hamman `_. +- New :py:meth:`~xarray.DataArray.rank` on arrays and datasets. Requires + bottleneck (:issue:`1731`). + By `0x0L `_. +- ``.dt`` accessor can now ceil, floor and round timestamps to specified frequency. + By `Deepak Cherian `_. + +**Plotting enhancements**: + +- :func:`xarray.plot.imshow` now handles RGB and RGBA images. + Saturation can be adjusted with ``vmin`` and ``vmax``, or with ``robust=True``. + By `Zac Hatfield-Dodds `_. +- :py:func:`~plot.contourf()` learned to contour 2D variables that have both a + 1D coordinate (e.g. time) and a 2D coordinate (e.g. depth as a function of + time) (:issue:`1737`). + By `Deepak Cherian `_. +- :py:func:`~plot()` rotates x-axis ticks if x-axis is time. + By `Deepak Cherian `_. +- :py:func:`~plot.line()` can draw multiple lines if provided with a + 2D variable. + By `Deepak Cherian `_. + +**Other enhancements**: + - Reduce methods such as :py:func:`DataArray.sum()` now handles object-type array. .. ipython:: python @@ -57,39 +94,17 @@ Enhancements By `Keisuke Fujii `_. - Added nodatavals attribute to DataArray when using :py:func:`~xarray.open_rasterio`. (:issue:`1736`). By `Alan Snow `_. -- :py:func:`~plot.contourf()` learned to contour 2D variables that have both a - 1D co-ordinate (e.g. time) and a 2D co-ordinate (e.g. depth as a function of - time) (:issue:`1737`). - By `Deepak Cherian `_. -- Added :py:meth:`DataArray.to_iris ` and :py:meth:`DataArray.from_iris ` for - converting data arrays to and from Iris_ Cubes with the same data and coordinates (:issue:`621` and :issue:`37`). - By `Neil Parley `_ and `Duncan Watson-Parris `_. - Use ``pandas.Grouper`` class in xarray resample methods rather than the deprecated ``pandas.TimeGrouper`` class (:issue:`1766`). By `Joe Hamman `_. -- Support for using `Zarr`_ as storage layer for xarray. (:issue:`1223`). - By `Ryan Abernathey `_ and - `Joe Hamman `_. -- Support for using `Zarr`_ as storage layer for xarray. - By `Ryan Abernathey `_. -- :func:`xarray.plot.imshow` now handles RGB and RGBA images. - Saturation can be adjusted with ``vmin`` and ``vmax``, or with ``robust=True``. - By `Zac Hatfield-Dodds `_. - Experimental support for parsing ENVI metadata to coordinates and attributes in :py:func:`xarray.open_rasterio`. By `Matti Eskelinen `_. -- :py:func:`~plot()` learned to rotate x-axis ticks if x-axis is time. - By `Deepak Cherian `_. -- :py:func:`~plot.line()` learned to draw multiple lines if provided with a - 2D variable. - By `Deepak Cherian `_. - Reduce memory usage when decoding a variable with a scale_factor, by converting 8-bit and 16-bit integers to float32 instead of float64 (:pull:`1840`), and keeping float16 and float32 as float32 (:issue:`1842`). Correspondingly, encoded variables may also be saved with a smaller dtype. By `Zac Hatfield-Dodds `_. -- `.dt` accessor can now ceil, floor and round timestamps to specified frequency. - By `Deepak Cherian `_. - Speed of reindexing/alignment with dask array is orders of magnitude faster when inserting missing values (:issue:`1847`). By `Stephan Hoyer `_. @@ -104,12 +119,6 @@ Enhancements .. _Iris: http://scitools.org.uk/iris -**New functions/methods** - -- New :py:meth:`~xarray.DataArray.rank` on arrays and datasets. Requires - bottleneck (:issue:`1731`). - By `0x0L `_. - Bug fixes ~~~~~~~~~ - Support indexing with a 0d-np.ndarray (:issue:`1921`). @@ -138,7 +147,7 @@ Bug fixes with size one in some dimension can now be plotted, which is good for exploring satellite imagery (:issue:`1780`). By `Zac Hatfield-Dodds `_. -- Fixed ``UnboundLocalError`` when opening netCDF file `` (:issue:`1781`). +- Fixed ``UnboundLocalError`` when opening netCDF file (:issue:`1781`). By `Stephan Hoyer `_. - The ``variables``, ``attrs``, and ``dimensions`` properties have been deprecated as part of a bug fix addressing an issue where backends were @@ -167,7 +176,7 @@ Bug fixes match pandas (:issue:`1847`). A combination of strings/numbers or unicode/bytes now promote to object dtype, instead of strings or unicode. By `Stephan Hoyer `_. - - Fixed bug where :py:meth:`~xarray.DataArray.isnull` was loading data +- Fixed bug where :py:meth:`~xarray.DataArray.isnull` was loading data stored as dask arrays (:issue:`1937`). By `Joe Hamman `_. From 30c4996b6c398f3bbba4b603dce5b88e011998bb Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 25 Feb 2018 16:23:03 -0800 Subject: [PATCH 029/282] Release v0.10.1 --- doc/whats-new.rst | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 7229c95490f..ef1283486dc 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,8 +27,8 @@ What's New .. _whats-new.0.10.1: -v0.10.1 (unreleased) --------------------- +v0.10.1 (25 February 2018) +-------------------------- The minor release includes a number of bug-fixes and backwards compatible enhancements. diff --git a/setup.py b/setup.py index ee717950dbe..3320d2ec4bc 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ MAJOR = 0 MINOR = 10 -MICRO = 0 -ISRELEASED = False +MICRO = 1 +ISRELEASED = True VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From 6c9116a8a023d245a895446d995b313a61153258 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 25 Feb 2018 16:27:12 -0800 Subject: [PATCH 030/282] Revert to dev version --- doc/whats-new.rst | 16 ++++++++++++++++ setup.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index ef1283486dc..89c4323d2eb 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,22 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ +.. _whats-new.0.10.2: + +v0.10.2 (unreleased) +-------------------- + +The minor release includes a number of bug-fixes and backwards compatible enhancements. + +Documentation +~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + .. _whats-new.0.10.1: v0.10.1 (25 February 2018) diff --git a/setup.py b/setup.py index 3320d2ec4bc..b5b56810bee 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ MAJOR = 0 MINOR = 10 MICRO = 1 -ISRELEASED = True +ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From 20207fd45280a3cb0f2ca3bdef693f47b1880dca Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 25 Feb 2018 16:32:12 -0800 Subject: [PATCH 031/282] Use newer numpydoc --- doc/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/environment.yml b/doc/environment.yml index 2758612c139..61a4687180c 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -6,7 +6,7 @@ dependencies: - python=3.5 - numpy=1.11.2 - pandas=0.21.0 - - numpydoc=0.6.0 + - numpydoc=0.7.0 - matplotlib=2.0.0 - seaborn=0.8 - dask=0.16.0 From f530e668fa50665245988be2a00748b9b3ccc0a8 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 25 Feb 2018 16:40:57 -0800 Subject: [PATCH 032/282] Use newer ipython for doc build --- doc/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/environment.yml b/doc/environment.yml index 61a4687180c..5be81e5a314 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -10,7 +10,7 @@ dependencies: - matplotlib=2.0.0 - seaborn=0.8 - dask=0.16.0 - - ipython=5.1.0 + - ipython=6.2.1 - sphinx=1.5 - netCDF4=1.3.1 - cartopy=0.15.1 From 519ff806f297b7dae616471421780fa949b223fd Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 26 Feb 2018 08:08:15 -0800 Subject: [PATCH 033/282] Add ReadTheDocs badge to README.rst (#1941) This will hopefully make it easier to keep track of our build status between releases. --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 37061e2c6ce..69ca02019c4 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,8 @@ xarray: N-D labeled arrays and datasets :target: https://ci.appveyor.com/project/shoyer/xray .. image:: https://coveralls.io/repos/pydata/xarray/badge.svg :target: https://coveralls.io/r/pydata/xarray +.. image:: https://readthedocs.org/projects/xray/badge/?version=latent + :target: http://xarray.pydata.org/ .. image:: https://img.shields.io/pypi/v/xarray.svg :target: https://pypi.python.org/pypi/xarray/ .. image:: https://zenodo.org/badge/13221727.svg From d8ccc7a999dce1a9ac205452e327bab5aa5f99f0 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Tue, 27 Feb 2018 10:13:45 +0900 Subject: [PATCH 034/282] Fix precision drop when indexing a datetime64 arrays. (#1942) * Fix precision drop when indexing a datetime64 arrays. * minor style fix. * Add a link to github issue in numpy/numpy --- doc/whats-new.rst | 3 +++ xarray/core/indexing.py | 4 ++++ xarray/tests/test_variable.py | 7 +++++++ 3 files changed, 14 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 89c4323d2eb..8b200103303 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -41,6 +41,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fix the precision drop after indexing datetime64 arrays (:issue:`1932`). + By `Keisuke Fujii `_. + .. _whats-new.0.10.1: v0.10.1 (25 February 2018) diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index e902aa8ffa3..49ae0c5b3af 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -911,6 +911,10 @@ def __getitem__(self, indexer): result = np.datetime64('NaT', 'ns') elif isinstance(result, timedelta): result = np.timedelta64(getattr(result, 'value', result), 'ns') + elif isinstance(result, pd.Timestamp): + # Work around for GH: pydata/xarray#1932 and numpy/numpy#10668 + # numpy fails to convert pd.Timestamp to np.datetime64[ns] + result = np.asarray(result.to_datetime64()) elif self.dtype != object: result = np.asarray(result, dtype=self.dtype) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index c8f74762683..f5125796a77 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1719,6 +1719,13 @@ def test_coordinate_alias(self): x = Coordinate('x', [1, 2, 3]) assert isinstance(x, IndexVariable) + def test_datetime64(self): + # GH:1932 Make sure indexing keeps precision + t = np.array([1518418799999986560, 1518418799999996560], + dtype='datetime64[ns]') + v = IndexVariable('t', t) + assert v[0].data == t[0] + # These tests make use of multi-dimensional variables, which are not valid # IndexVariable objects: @pytest.mark.xfail From 243093cf814ffaae2a9ce08215632500fbebcf52 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Tue, 27 Feb 2018 13:27:23 +0900 Subject: [PATCH 035/282] Fix rtd link on readme (#1943) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 69ca02019c4..40491680c3f 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ xarray: N-D labeled arrays and datasets :target: https://ci.appveyor.com/project/shoyer/xray .. image:: https://coveralls.io/repos/pydata/xarray/badge.svg :target: https://coveralls.io/r/pydata/xarray -.. image:: https://readthedocs.org/projects/xray/badge/?version=latent +.. image:: https://readthedocs.org/projects/xray/badge/?version=latest :target: http://xarray.pydata.org/ .. image:: https://img.shields.io/pypi/v/xarray.svg :target: https://pypi.python.org/pypi/xarray/ From 4ee244078ea90084624c1b6d006f50285f8f2d21 Mon Sep 17 00:00:00 2001 From: Benoit Bovy Date: Tue, 27 Feb 2018 20:04:24 +0100 Subject: [PATCH 036/282] DOC: add main sections to toc (#1946) * add main sections to toc * move whats new to "help and references" section --- doc/index.rst | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index e9df085169a..f4f036b8e58 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,4 +1,3 @@ - .. image:: _static/dataset-diagram-logo.png :width: 300 px :align: center @@ -32,14 +31,42 @@ describing scientific data in widespread use in the Earth sciences: Documentation ------------- +**Getting Started** + +* :doc:`why-xarray` +* :doc:`faq` +* :doc:`examples` +* :doc:`installing` + .. toctree:: :maxdepth: 1 + :hidden: + :caption: Getting Started - whats-new why-xarray faq examples installing + +**User Guide** + +* :doc:`data-structures` +* :doc:`indexing` +* :doc:`computation` +* :doc:`groupby` +* :doc:`reshaping` +* :doc:`combining` +* :doc:`time-series` +* :doc:`pandas` +* :doc:`io` +* :doc:`dask` +* :doc:`plotting` + +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: User Guide + data-structures indexing computation @@ -51,6 +78,20 @@ Documentation io dask plotting + +**Help & reference** + +* :doc:`whats-new` +* :doc:`api` +* :doc:`internals` +* :doc:`contributing` + +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Help & reference + + whats-new api internals contributing From fd2e542d61c5d805a3aa159d96cde279f4c5f0b8 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 27 Feb 2018 11:12:52 -0800 Subject: [PATCH 037/282] Add seaborn import to toy weather data example. (#1945) * Add seaborn import to toy weather data example. Fixes GH1944 It looks like this got inadvertently removed with the flake8 fix in GH1925. * Tweak stickler config * Tweak stickler again * temp tweak * more tweaking of config * more config tweak * try again * once again * alt disable * add comment * final line break --- .stickler.yml | 8 ++++++-- doc/examples/_code/weather_data_setup.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.stickler.yml b/.stickler.yml index d708ec10ec1..db8f5f254e9 100644 --- a/.stickler.yml +++ b/.stickler.yml @@ -3,6 +3,10 @@ linters: max-line-length: 79 fixer: false ignore: I002 - exclude: - - doc/* + # stickler doesn't support 'exclude' for flake8 properly, so we disable it + # below with files.ignore: + # https://github.com/markstory/lint-review/issues/184 py3k: +files: + ignore: + - doc/**/*.py diff --git a/doc/examples/_code/weather_data_setup.py b/doc/examples/_code/weather_data_setup.py index b370aea00f2..e07c6a865bb 100644 --- a/doc/examples/_code/weather_data_setup.py +++ b/doc/examples/_code/weather_data_setup.py @@ -1,6 +1,7 @@ import xarray as xr import numpy as np import pandas as pd +import seaborn as sns # pandas aware plotting library np.random.seed(123) From 0e73e240107caee3ffd1a1149f0150c390d43251 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Tue, 27 Feb 2018 14:33:34 -0500 Subject: [PATCH 038/282] isort (#1924) * isort configs * all files sorted * default to thirdparty * sorting after defaulting to third party * don't ignore sorting errors in Stickler (though I don't think it'll check anyway!) * lots of gaps? * double negative * @stickler-ci * isort home stretch * unsure why this is being picked up now * ok fun is fading now * Fixing style errors. * random char * lint --- asv_bench/benchmarks/dataarray_missing.py | 12 +++--- asv_bench/benchmarks/dataset_io.py | 12 +++--- asv_bench/benchmarks/indexing.py | 8 ++-- asv_bench/benchmarks/reindexing.py | 5 +-- doc/conf.py | 11 +++-- doc/examples/_code/weather_data_setup.py | 3 +- doc/gallery/plot_cartopy_facetgrid.py | 3 +- doc/gallery/plot_colorbar_center.py | 3 +- doc/gallery/plot_lines_from_2d.py | 3 +- doc/gallery/plot_rasterio.py | 5 ++- doc/gallery/plot_rasterio_rgb.py | 4 +- setup.cfg | 7 +++- setup.py | 2 +- xarray/backends/api.py | 12 +++--- xarray/backends/common.py | 14 +++---- xarray/backends/h5netcdf_.py | 15 ++++--- xarray/backends/memory.py | 8 ++-- xarray/backends/netCDF4_.py | 20 ++++----- xarray/backends/netcdf3.py | 10 ++--- xarray/backends/pydap_.py | 8 ++-- xarray/backends/pynio_.py | 9 ++-- xarray/backends/rasterio_.py | 4 +- xarray/backends/scipy_.py | 18 ++++---- xarray/backends/zarr.py | 15 +++---- xarray/coding/times.py | 22 +++++----- xarray/coding/variables.py | 11 ++--- xarray/conventions.py | 8 +--- xarray/convert.py | 8 ++-- xarray/core/accessors.py | 10 ++--- xarray/core/alignment.py | 11 +++-- xarray/core/combine.py | 11 +++-- xarray/core/common.py | 18 ++++---- xarray/core/computation.py | 4 +- xarray/core/coordinates.py | 10 ++--- xarray/core/dataarray.py | 30 +++++--------- xarray/core/dataset.py | 50 ++++++++++------------- xarray/core/dtypes.py | 4 +- xarray/core/duck_array_ops.py | 11 ++--- xarray/core/extensions.py | 5 +-- xarray/core/formatting.py | 14 +++---- xarray/core/groupby.py | 21 ++++------ xarray/core/indexing.py | 18 ++++---- xarray/core/merge.py | 6 +-- xarray/core/missing.py | 9 ++-- xarray/core/npcompat.py | 5 +-- xarray/core/nputils.py | 8 ++-- xarray/core/ops.py | 9 ++-- xarray/core/options.py | 5 +-- xarray/core/pycompat.py | 5 +-- xarray/core/resample.py | 6 +-- xarray/core/rolling.py | 17 ++++---- xarray/core/utils.py | 11 +++-- xarray/core/variable.py | 30 ++++++-------- xarray/plot/facetgrid.py | 15 +++---- xarray/plot/plot.py | 15 +++---- xarray/plot/utils.py | 8 ++-- xarray/testing.py | 4 +- xarray/tests/test_accessors.py | 11 +++-- xarray/tests/test_backends.py | 34 ++++++++------- xarray/tests/test_coding.py | 5 +-- xarray/tests/test_coding_times.py | 10 ++--- xarray/tests/test_combine.py | 17 ++++---- xarray/tests/test_computation.py | 13 +++--- xarray/tests/test_conventions.py | 23 +++++------ xarray/tests/test_dask.py | 22 +++++----- xarray/tests/test_dataarray.py | 27 ++++++------ xarray/tests/test_dataset.py | 45 ++++++++++---------- xarray/tests/test_distributed.py | 26 ++++++++---- xarray/tests/test_dtypes.py | 4 +- xarray/tests/test_duck_array_ops.py | 21 +++++----- xarray/tests/test_extensions.py | 16 ++++---- xarray/tests/test_formatting.py | 5 +-- xarray/tests/test_groupby.py | 9 ++-- xarray/tests/test_indexing.py | 17 ++++---- xarray/tests/test_merge.py | 12 +++--- xarray/tests/test_missing.py | 20 ++++----- xarray/tests/test_nputils.py | 2 +- xarray/tests/test_options.py | 7 ++-- xarray/tests/test_plot.py | 33 ++++++++------- xarray/tests/test_testing.py | 4 +- xarray/tests/test_tutorial.py | 8 ++-- xarray/tests/test_ufuncs.py | 10 ++--- xarray/tests/test_utils.py | 9 ++-- xarray/tests/test_variable.py | 33 +++++++-------- xarray/tutorial.py | 6 +-- xarray/ufuncs.py | 12 ++---- xarray/util/print_versions.py | 10 +++-- 87 files changed, 492 insertions(+), 609 deletions(-) diff --git a/asv_bench/benchmarks/dataarray_missing.py b/asv_bench/benchmarks/dataarray_missing.py index e0127bb7da2..29a9e78f82c 100644 --- a/asv_bench/benchmarks/dataarray_missing.py +++ b/asv_bench/benchmarks/dataarray_missing.py @@ -1,18 +1,16 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import pandas as pd +import xarray as xr + +from . import randn, requires_dask + try: import dask # noqa except ImportError: pass -import xarray as xr - -from . import randn, requires_dask - def make_bench_data(shape, frac_nan, chunks): vals = randn(shape, frac_nan) diff --git a/asv_bench/benchmarks/dataset_io.py b/asv_bench/benchmarks/dataset_io.py index d7766d99a3d..de6c34b5af3 100644 --- a/asv_bench/benchmarks/dataset_io.py +++ b/asv_bench/benchmarks/dataset_io.py @@ -1,20 +1,18 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np import pandas as pd +import xarray as xr + +from . import randn, requires_dask + try: import dask import dask.multiprocessing except ImportError: pass -import xarray as xr - -from . import randn, requires_dask - class IOSingleNetCDF(object): """ diff --git a/asv_bench/benchmarks/indexing.py b/asv_bench/benchmarks/indexing.py index 9a41c6cf0e7..54262b12a19 100644 --- a/asv_bench/benchmarks/indexing.py +++ b/asv_bench/benchmarks/indexing.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np import pandas as pd -import xarray as xr -from . import randn, randint, requires_dask +import xarray as xr +from . import randint, randn, requires_dask nx = 3000 ny = 2000 diff --git a/asv_bench/benchmarks/reindexing.py b/asv_bench/benchmarks/reindexing.py index 0f28eaa4cee..28e14d52e89 100644 --- a/asv_bench/benchmarks/reindexing.py +++ b/asv_bench/benchmarks/reindexing.py @@ -1,8 +1,7 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np + import xarray as xr from . import requires_dask diff --git a/doc/conf.py b/doc/conf.py index f4c7d7058e5..2f6849fd0bd 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,14 +11,14 @@ # # All configuration values have a default; values that are commented out # serve to show the default. -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function -import sys -import os import datetime import importlib +import os +import sys + +import xarray allowed_failures = set() @@ -44,7 +44,6 @@ 'gallery/plot_rasterio_rgb.py', 'gallery/plot_rasterio.py']) -import xarray print("xarray: %s, %s" % (xarray.__version__, xarray.__file__)) # -- General configuration ------------------------------------------------ diff --git a/doc/examples/_code/weather_data_setup.py b/doc/examples/_code/weather_data_setup.py index e07c6a865bb..89470542d5a 100644 --- a/doc/examples/_code/weather_data_setup.py +++ b/doc/examples/_code/weather_data_setup.py @@ -1,8 +1,9 @@ -import xarray as xr import numpy as np import pandas as pd import seaborn as sns # pandas aware plotting library +import xarray as xr + np.random.seed(123) times = pd.date_range('2000-01-01', '2001-12-31', name='time') diff --git a/doc/gallery/plot_cartopy_facetgrid.py b/doc/gallery/plot_cartopy_facetgrid.py index 10d782ca41f..3eded115263 100644 --- a/doc/gallery/plot_cartopy_facetgrid.py +++ b/doc/gallery/plot_cartopy_facetgrid.py @@ -16,10 +16,11 @@ from __future__ import division -import xarray as xr import cartopy.crs as ccrs import matplotlib.pyplot as plt +import xarray as xr + # Load the data ds = xr.tutorial.load_dataset('air_temperature') air = ds.air.isel(time=[0, 724]) - 273.15 diff --git a/doc/gallery/plot_colorbar_center.py b/doc/gallery/plot_colorbar_center.py index ce83b4f05cf..4818b737632 100644 --- a/doc/gallery/plot_colorbar_center.py +++ b/doc/gallery/plot_colorbar_center.py @@ -8,9 +8,10 @@ """ -import xarray as xr import matplotlib.pyplot as plt +import xarray as xr + # Load the data ds = xr.tutorial.load_dataset('air_temperature') air = ds.air.isel(time=0) diff --git a/doc/gallery/plot_lines_from_2d.py b/doc/gallery/plot_lines_from_2d.py index 1e5875ea70e..93d7770238e 100644 --- a/doc/gallery/plot_lines_from_2d.py +++ b/doc/gallery/plot_lines_from_2d.py @@ -12,9 +12,10 @@ """ -import xarray as xr import matplotlib.pyplot as plt +import xarray as xr + # Load the data ds = xr.tutorial.load_dataset('air_temperature') air = ds.air - 273.15 # to celsius diff --git a/doc/gallery/plot_rasterio.py b/doc/gallery/plot_rasterio.py index d5234950702..98801990af3 100644 --- a/doc/gallery/plot_rasterio.py +++ b/doc/gallery/plot_rasterio.py @@ -18,12 +18,13 @@ import os import urllib.request -import numpy as np -import xarray as xr + import cartopy.crs as ccrs import matplotlib.pyplot as plt +import numpy as np from rasterio.warp import transform +import xarray as xr # Download the file from rasterio's repository url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif' diff --git a/doc/gallery/plot_rasterio_rgb.py b/doc/gallery/plot_rasterio_rgb.py index ec2bbe63218..2733bf149e5 100644 --- a/doc/gallery/plot_rasterio_rgb.py +++ b/doc/gallery/plot_rasterio_rgb.py @@ -15,10 +15,12 @@ import os import urllib.request -import xarray as xr + import cartopy.crs as ccrs import matplotlib.pyplot as plt +import xarray as xr + # Download the file from rasterio's repository url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif' urllib.request.urlretrieve(url, 'RGB.byte.tif') diff --git a/setup.cfg b/setup.cfg index c7df2b7fc74..fe6e63a5080 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,11 @@ python_files=test_*.py [flake8] max-line-length=79 -ignore=I002 +ignore= exclude= doc/ + +[isort] +default_section=THIRDPARTY +known_first_party=xarray +multi_line_output=4 diff --git a/setup.py b/setup.py index b5b56810bee..e81d3d2600b 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import sys import warnings -from setuptools import setup, find_packages +from setuptools import find_packages, setup MAJOR = 0 MINOR = 10 diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 1effdf18dac..9d0b95c8c81 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -1,20 +1,18 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import os.path from glob import glob from io import BytesIO from numbers import Number - import numpy as np -from .. import backends, conventions, Dataset -from .common import ArrayWriter, GLOBAL_LOCK +from .. import Dataset, backends, conventions from ..core import indexing from ..core.combine import auto_combine -from ..core.utils import close_on_error, is_remote_uri from ..core.pycompat import basestring, path_type +from ..core.utils import close_on_error, is_remote_uri +from .common import GLOBAL_LOCK, ArrayWriter DATAARRAY_NAME = '__xarray_dataarray_name__' DATAARRAY_VARIABLE = '__xarray_dataarray_variable__' diff --git a/xarray/backends/common.py b/xarray/backends/common.py index 157ee494067..d91cedbbda3 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -1,18 +1,18 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import numpy as np +from __future__ import absolute_import, division, print_function + +import contextlib import logging import time import traceback -import contextlib -from collections import Mapping, OrderedDict import warnings +from collections import Mapping, OrderedDict + +import numpy as np from ..conventions import cf_encoder from ..core import indexing +from ..core.pycompat import dask_array_type, iteritems from ..core.utils import FrozenOrderedDict, NdimSizeLenMixin -from ..core.pycompat import iteritems, dask_array_type try: from dask.utils import SerializableLock as Lock diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index cba1d33115f..4e70c8858c3 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -1,18 +1,17 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools import numpy as np from .. import Variable from ..core import indexing +from ..core.pycompat import OrderedDict, bytes_type, iteritems, unicode_type from ..core.utils import FrozenOrderedDict, close_on_error -from ..core.pycompat import iteritems, bytes_type, unicode_type, OrderedDict - -from .common import WritableCFDataStore, DataStorePickleMixin, find_root -from .netCDF4_ import (_nc4_group, _encode_nc4_variable, _get_datatype, - _extract_nc4_variable_encoding, BaseNetCDF4Array) +from .common import DataStorePickleMixin, WritableCFDataStore, find_root +from .netCDF4_ import ( + BaseNetCDF4Array, _encode_nc4_variable, _extract_nc4_variable_encoding, + _get_datatype, _nc4_group) class H5NetCDFArrayWrapper(BaseNetCDF4Array): diff --git a/xarray/backends/memory.py b/xarray/backends/memory.py index 8c09277b2d0..69a54133716 100644 --- a/xarray/backends/memory.py +++ b/xarray/backends/memory.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import copy import numpy as np -from ..core.variable import Variable from ..core.pycompat import OrderedDict - +from ..core.variable import Variable from .common import AbstractWritableDataStore diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 3f3364dec56..313539bd6bc 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools import operator import warnings @@ -8,16 +7,15 @@ import numpy as np -from .. import conventions -from .. import Variable +from .. import Variable, conventions from ..conventions import pop_to from ..core import indexing -from ..core.utils import (FrozenOrderedDict, close_on_error, is_remote_uri) -from ..core.pycompat import iteritems, basestring, OrderedDict, PY3, suppress - -from .common import (WritableCFDataStore, robust_getitem, BackendArray, - DataStorePickleMixin, find_root) -from .netcdf3 import (encode_nc3_attr_value, encode_nc3_variable) +from ..core.pycompat import PY3, OrderedDict, basestring, iteritems, suppress +from ..core.utils import FrozenOrderedDict, close_on_error, is_remote_uri +from .common import ( + BackendArray, DataStorePickleMixin, WritableCFDataStore, find_root, + robust_getitem) +from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable # This lookup table maps from dtype.byteorder to a readable endian # string used by netCDF4. diff --git a/xarray/backends/netcdf3.py b/xarray/backends/netcdf3.py index 7aa054bc119..f0ded98d954 100644 --- a/xarray/backends/netcdf3.py +++ b/xarray/backends/netcdf3.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import unicodedata import numpy as np -from .. import conventions, Variable -from ..core.pycompat import basestring, unicode_type, OrderedDict - +from .. import Variable, conventions +from ..core.pycompat import OrderedDict, basestring, unicode_type # Special characters that are permitted in netCDF names except in the # 0th position of the string diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index 297d96e47f4..a16b1ddcbc8 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import numpy as np from .. import Variable -from ..core.utils import FrozenOrderedDict, Frozen, is_dict_like from ..core import indexing from ..core.pycompat import integer_types - +from ..core.utils import Frozen, FrozenOrderedDict, is_dict_like from .common import AbstractDataStore, BackendArray, robust_getitem diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 37f1db1f6a7..30969fcd9a0 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -1,16 +1,13 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import numpy as np from .. import Variable -from ..core.utils import (FrozenOrderedDict, Frozen) from ..core import indexing - -from .common import AbstractDataStore, DataStorePickleMixin, BackendArray +from ..core.utils import Frozen, FrozenOrderedDict +from .common import AbstractDataStore, BackendArray, DataStorePickleMixin class NioArrayWrapper(BackendArray): diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index b3b94c86c3c..8777f6e7053 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -2,12 +2,14 @@ import warnings from collections import OrderedDict from distutils.version import LooseVersion + import numpy as np from .. import DataArray -from ..core.utils import is_scalar from ..core import indexing +from ..core.utils import is_scalar from .common import BackendArray + try: from dask.utils import SerializableLock as Lock except ImportError: diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index a608cff8eb5..a0765fe27bd 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -1,20 +1,18 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools +import warnings from io import BytesIO import numpy as np -import warnings from .. import Variable -from ..core.pycompat import iteritems, OrderedDict, basestring -from ..core.utils import (Frozen, FrozenOrderedDict) from ..core.indexing import NumpyIndexingAdapter - -from .common import WritableCFDataStore, DataStorePickleMixin, BackendArray -from .netcdf3 import (is_valid_nc3_name, encode_nc3_attr_value, - encode_nc3_variable) +from ..core.pycompat import OrderedDict, basestring, iteritems +from ..core.utils import Frozen, FrozenOrderedDict +from .common import BackendArray, DataStorePickleMixin, WritableCFDataStore +from .netcdf3 import ( + encode_nc3_attr_value, encode_nc3_variable, is_valid_nc3_name) def _decode_string(s): diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 12149026ac3..b0323b51f17 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1,18 +1,15 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from itertools import product +from __future__ import absolute_import, division, print_function + from base64 import b64encode +from itertools import product import numpy as np -from .. import coding -from .. import Variable +from .. import Variable, coding, conventions from ..core import indexing +from ..core.pycompat import OrderedDict, integer_types, iteritems from ..core.utils import FrozenOrderedDict, HiddenKeyDict -from ..core.pycompat import iteritems, OrderedDict, integer_types -from .common import AbstractWritableDataStore, BackendArray, ArrayWriter -from .. import conventions +from .common import AbstractWritableDataStore, ArrayWriter, BackendArray # need some special secret attributes to tell us the dimensions _DIMENSION_KEY = '_ARRAY_DIMENSIONS' diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 28afc46f660..1bb4e31ae7e 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -1,6 +1,4 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import re import traceback @@ -9,21 +7,21 @@ from functools import partial import numpy as np - import pandas as pd -try: - from pandas.errors import OutOfBoundsDatetime -except ImportError: - # pandas < 0.20 - from pandas.tslib import OutOfBoundsDatetime -from .variables import (SerializationWarning, VariableCoder, - lazy_elemwise_func, pop_to, safe_setitem, - unpack_for_decoding, unpack_for_encoding) from ..core import indexing from ..core.formatting import first_n_items, format_timestamp, last_item from ..core.pycompat import PY3 from ..core.variable import Variable +from .variables import ( + SerializationWarning, VariableCoder, lazy_elemwise_func, pop_to, + safe_setitem, unpack_for_decoding, unpack_for_encoding) + +try: + from pandas.errors import OutOfBoundsDatetime +except ImportError: + # pandas < 0.20 + from pandas.tslib import OutOfBoundsDatetime # standard calendars recognized by netcdftime diff --git a/xarray/coding/variables.py b/xarray/coding/variables.py index 5d32970e2ed..ced535643a5 100644 --- a/xarray/coding/variables.py +++ b/xarray/coding/variables.py @@ -1,18 +1,13 @@ """Coders for individual Variable objects.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from functools import partial import warnings +from functools import partial import numpy as np import pandas as pd -from ..core import dtypes -from ..core import duck_array_ops -from ..core import indexing -from ..core import utils +from ..core import dtypes, duck_array_ops, indexing, utils from ..core.pycompat import dask_array_type from ..core.variable import Variable diff --git a/xarray/conventions.py b/xarray/conventions.py index fe75d9e3e6a..5bcbd83ee90 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -1,16 +1,12 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import warnings from collections import defaultdict import numpy as np - import pandas as pd -from .coding import times -from .coding import variables +from .coding import times, variables from .coding.variables import SerializationWarning from .core import duck_array_ops, indexing from .core.pycompat import OrderedDict, basestring, iteritems diff --git a/xarray/convert.py b/xarray/convert.py index caf665b421d..a6defd083bf 100644 --- a/xarray/convert.py +++ b/xarray/convert.py @@ -1,16 +1,14 @@ """Functions for converting to and from xarray objects """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np from .coding.times import CFDatetimeCoder, CFTimedeltaCoder +from .conventions import decode_cf from .core.dataarray import DataArray -from .core.pycompat import OrderedDict, range from .core.dtypes import get_fill_value -from .conventions import decode_cf +from .core.pycompat import OrderedDict, range cdms2_ignored_attrs = {'name', 'tileIndex'} iris_forbidden_keys = {'standard_name', 'long_name', 'units', 'bounds', 'axis', diff --git a/xarray/core/accessors.py b/xarray/core/accessors.py index b3e7e1ff9a1..52d9e6db408 100644 --- a/xarray/core/accessors.py +++ b/xarray/core/accessors.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from .dtypes import is_datetime_like -from .pycompat import dask_array_type +from __future__ import absolute_import, division, print_function import numpy as np import pandas as pd +from .dtypes import is_datetime_like +from .pycompat import dask_array_type + def _season_from_months(months): """Compute season (DJF, MAM, JJA, SON) from month ordinal diff --git a/xarray/core/alignment.py b/xarray/core/alignment.py index 99dde45b892..b0d2a49c29f 100644 --- a/xarray/core/alignment.py +++ b/xarray/core/alignment.py @@ -1,17 +1,16 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools import operator -from collections import defaultdict import warnings +from collections import defaultdict import numpy as np from . import utils from .indexing import get_indexer_nd -from .pycompat import iteritems, OrderedDict, suppress -from .utils import is_full_slice, is_dict_like +from .pycompat import OrderedDict, iteritems, suppress +from .utils import is_dict_like, is_full_slice from .variable import IndexVariable diff --git a/xarray/core/combine.py b/xarray/core/combine.py index b14d085f383..149009689e9 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import warnings import pandas as pd @@ -8,9 +7,9 @@ from . import utils from .alignment import align from .merge import merge -from .pycompat import iteritems, OrderedDict, basestring -from .variable import Variable, as_variable, IndexVariable, \ - concat as concat_vars +from .pycompat import OrderedDict, basestring, iteritems +from .variable import IndexVariable, Variable, as_variable +from .variable import concat as concat_vars def concat(objs, dim=None, data_vars='all', coords='different', diff --git a/xarray/core/common.py b/xarray/core/common.py index 094e8350eb6..d521e7ae5c2 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -1,15 +1,13 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + +import warnings + import numpy as np import pandas as pd -import warnings -from .pycompat import basestring, suppress, dask_array_type, OrderedDict -from . import dtypes -from . import formatting -from . import ops -from .utils import SortedKeysDict, not_implemented, Frozen +from . import dtypes, formatting, ops +from .pycompat import OrderedDict, basestring, dask_array_type, suppress +from .utils import Frozen, SortedKeysDict, not_implemented class ImplementsArrayReduce(object): @@ -214,7 +212,7 @@ def _ipython_key_completions_(self): def get_squeeze_dims(xarray_obj, dim, axis=None): """Get a list of dimensions to squeeze out. """ - if not dim is None and not axis is None: + if dim is not None and axis is not None: raise ValueError('cannot use both parameters `axis` and `dim`') if dim is None and axis is None: diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 5dd9fe78c56..b7590ab6b4b 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -9,14 +9,12 @@ import numpy as np -from . import duck_array_ops -from . import utils +from . import duck_array_ops, utils from .alignment import deep_align from .merge import expand_and_merge_variables from .pycompat import OrderedDict, dask_array_type from .utils import is_dict_like - _DEFAULT_FROZEN_SET = frozenset() _NO_FILL_VALUE = utils.ReprObject('') _DEFAULT_NAME = utils.ReprObject('') diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index 60c01e8be72..522206f72b0 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -1,15 +1,15 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + from collections import Mapping from contextlib import contextmanager + import pandas as pd from . import formatting, indexing -from .utils import Frozen from .merge import ( - merge_coords, expand_and_merge_variables, merge_coords_for_inplace_math) + expand_and_merge_variables, merge_coords, merge_coords_for_inplace_math) from .pycompat import OrderedDict +from .utils import Frozen from .variable import Variable diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 8e1ec8ab7b8..8c0360df8a9 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1,35 +1,27 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools import warnings import numpy as np import pandas as pd +from . import duck_array_ops, groupby, indexing, ops, resample, rolling, utils from ..plot.plot import _PlotMethods - -from . import duck_array_ops -from . import indexing -from . import groupby -from . import resample -from . import rolling -from . import ops -from . import utils from .accessors import DatetimeAccessor from .alignment import align, reindex_like_indexers from .common import AbstractArray, BaseDataObject -from .coordinates import (DataArrayCoordinates, LevelCoordinatesSource, - Indexes, assert_coordinate_consistent, - remap_label_indexers) +from .coordinates import ( + DataArrayCoordinates, Indexes, LevelCoordinatesSource, + assert_coordinate_consistent, remap_label_indexers) from .dataset import Dataset, merge_indexes, split_indexes -from .pycompat import iteritems, basestring, OrderedDict, zip, range -from .variable import (as_variable, Variable, as_compatible_data, - IndexVariable, - assert_unique_multiindex_level_names) from .formatting import format_item -from .utils import decode_numpy_dict_values, ensure_us_time_resolution from .options import OPTIONS +from .pycompat import OrderedDict, basestring, iteritems, range, zip +from .utils import decode_numpy_dict_values, ensure_us_time_resolution +from .variable import ( + IndexVariable, Variable, as_compatible_data, as_variable, + assert_unique_multiindex_level_names) def _infer_coords_and_dims(shape, coords, dims): diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 2e5c9a84b31..2a2c4e382ce 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1,43 +1,37 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools +import sys +import warnings from collections import Mapping, defaultdict from distutils.version import LooseVersion from numbers import Number -import warnings - -import sys import numpy as np import pandas as pd -from . import ops -from . import utils -from . import groupby -from . import resample -from . import rolling -from . import indexing -from . import alignment -from . import formatting -from . import duck_array_ops +import xarray as xr + +from . import ( + alignment, duck_array_ops, formatting, groupby, indexing, ops, resample, + rolling, utils) from .. import conventions from .alignment import align -from .coordinates import (DatasetCoordinates, LevelCoordinatesSource, Indexes, - assert_coordinate_consistent, remap_label_indexers) -from .common import ImplementsDatasetReduce, BaseDataObject +from .common import BaseDataObject, ImplementsDatasetReduce +from .coordinates import ( + DatasetCoordinates, Indexes, LevelCoordinatesSource, + assert_coordinate_consistent, remap_label_indexers) from .dtypes import is_datetime_like -from .merge import (dataset_update_method, dataset_merge_method, - merge_data_and_coords, merge_variables) -from .utils import (Frozen, SortedKeysDict, maybe_wrap_array, hashable, - decode_numpy_dict_values, ensure_us_time_resolution) -from .variable import (Variable, as_variable, IndexVariable, - broadcast_variables) -from .pycompat import (iteritems, basestring, OrderedDict, - integer_types, dask_array_type, range) +from .merge import ( + dataset_merge_method, dataset_update_method, merge_data_and_coords, + merge_variables) from .options import OPTIONS - -import xarray as xr +from .pycompat import ( + OrderedDict, basestring, dask_array_type, integer_types, iteritems, range) +from .utils import ( + Frozen, SortedKeysDict, decode_numpy_dict_values, + ensure_us_time_resolution, hashable, maybe_wrap_array) +from .variable import IndexVariable, Variable, as_variable, broadcast_variables # list of attributes of pd.DatetimeIndex that are ndarrays of time info _DATETIMEINDEX_COMPONENTS = ['year', 'month', 'day', 'hour', 'minute', diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 8dac39612e4..7326b936e2e 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -1,8 +1,8 @@ -import numpy as np import functools -from . import utils +import numpy as np +from . import utils # Use as a sentinel value to indicate a dtype appropriate NA value. NA = utils.ReprObject('') diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 6f5548800a2..1a1bcf36c56 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -3,22 +3,19 @@ Currently, this means Dask or NumPy arrays. None of these functions should accept or return xarray objects. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from functools import partial import contextlib import inspect import warnings +from functools import partial import numpy as np import pandas as pd -from . import npcompat -from . import dtypes -from .pycompat import dask_array_type +from . import dtypes, npcompat from .nputils import nanfirst, nanlast +from .pycompat import dask_array_type try: import bottleneck as bn diff --git a/xarray/core/extensions.py b/xarray/core/extensions.py index 90639e47f43..8070e07a5ef 100644 --- a/xarray/core/extensions.py +++ b/xarray/core/extensions.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import traceback import warnings diff --git a/xarray/core/formatting.py b/xarray/core/formatting.py index 83f8e2719d6..2009df3b2d1 100644 --- a/xarray/core/formatting.py +++ b/xarray/core/formatting.py @@ -4,24 +4,24 @@ be returned by the __unicode__ special method. We use ReprMixin to provide the __repr__ method so that things can work on Python 2. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import contextlib -from datetime import datetime, timedelta import functools +from datetime import datetime, timedelta import numpy as np import pandas as pd + +from .options import OPTIONS +from .pycompat import PY2, bytes_type, dask_array_type, unicode_type + try: from pandas.errors import OutOfBoundsDatetime except ImportError: # pandas < 0.20 from pandas.tslib import OutOfBoundsDatetime -from .options import OPTIONS -from .pycompat import PY2, unicode_type, bytes_type, dask_array_type - def pretty_print(x, numchars): """Given an object `x`, call `str(x)` and format the returned string so diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index c4b25741d5b..b722a01ec46 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -1,21 +1,16 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools + import numpy as np import pandas as pd -from . import dtypes -from . import duck_array_ops -from . import nputils -from . import ops +from . import dtypes, duck_array_ops, nputils, ops from .combine import concat -from .common import ( - ImplementsArrayReduce, ImplementsDatasetReduce, -) -from .pycompat import range, zip, integer_types -from .utils import hashable, peek_at, maybe_wrap_array, safe_cast_to_index -from .variable import as_variable, Variable, IndexVariable +from .common import ImplementsArrayReduce, ImplementsDatasetReduce +from .pycompat import integer_types, range, zip +from .utils import hashable, maybe_wrap_array, peek_at, safe_cast_to_index +from .variable import IndexVariable, Variable, as_variable def unique_value_groups(ar, sort=True): diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 49ae0c5b3af..0d55eed894e 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -1,18 +1,16 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from datetime import timedelta -from collections import defaultdict, Hashable +from __future__ import absolute_import, division, print_function + import functools import operator +from collections import Hashable, defaultdict +from datetime import timedelta + import numpy as np import pandas as pd -from . import nputils -from . import utils -from . import duck_array_ops -from .pycompat import (iteritems, range, integer_types, dask_array_type, - suppress) +from . import duck_array_ops, nputils, utils +from .pycompat import ( + dask_array_type, integer_types, iteritems, range, suppress) from .utils import is_dict_like diff --git a/xarray/core/merge.py b/xarray/core/merge.py index c5e643adb0d..7069ca9d96b 100644 --- a/xarray/core/merge.py +++ b/xarray/core/merge.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import pandas as pd from .alignment import deep_align @@ -8,7 +7,6 @@ from .utils import Frozen from .variable import as_variable, assert_unique_multiindex_level_names - PANDAS_TYPES = (pd.Series, pd.DataFrame, pd.Panel) _VALID_COMPAT = Frozen({'identical': 0, diff --git a/xarray/core/missing.py b/xarray/core/missing.py index e26e976a11b..e58d74f4c0d 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -1,6 +1,4 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from collections import Iterable from functools import partial @@ -8,11 +6,10 @@ import numpy as np import pandas as pd - -from .pycompat import iteritems from .computation import apply_ufunc -from .utils import is_scalar from .npcompat import flip +from .pycompat import iteritems +from .utils import is_scalar class BaseInterpolator(object): diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index bbe7b745621..df1e955518c 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import numpy as np try: diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 8ac04752e85..c781ca65a69 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -1,9 +1,9 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + +import warnings + import numpy as np import pandas as pd -import warnings def _validate_axis(data, axis): diff --git a/xarray/core/ops.py b/xarray/core/ops.py index f9e1e3ba355..32b31010b5f 100644 --- a/xarray/core/ops.py +++ b/xarray/core/ops.py @@ -5,18 +5,15 @@ functions. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import operator import numpy as np -from . import dtypes -from . import duck_array_ops -from .pycompat import PY3 +from . import dtypes, duck_array_ops from .nputils import array_eq, array_ne +from .pycompat import PY3 try: import bottleneck as bn diff --git a/xarray/core/options.py b/xarray/core/options.py index 9f06f8dbbae..b2968a2a02f 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -1,7 +1,4 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - +from __future__ import absolute_import, division, print_function OPTIONS = { 'display_width': 80, diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 19c16e445a6..df7781ca9c1 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -1,8 +1,7 @@ # flake8: noqa -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import sys import numpy as np diff --git a/xarray/core/resample.py b/xarray/core/resample.py index 78fd39d3245..4933a09b257 100644 --- a/xarray/core/resample.py +++ b/xarray/core/resample.py @@ -1,10 +1,8 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from . import ops from .groupby import DataArrayGroupBy, DatasetGroupBy -from .pycompat import dask_array_type, OrderedDict +from .pycompat import OrderedDict, dask_array_type RESAMPLE_DIM = '__resample_dim__' diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 8209e70e5a8..4bb020cebeb 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -1,16 +1,17 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import numpy as np +from __future__ import absolute_import, division, print_function + import warnings from distutils.version import LooseVersion -from .pycompat import OrderedDict, zip, dask_array_type -from .common import full_like +import numpy as np + from .combine import concat -from .ops import (inject_bottleneck_rolling_methods, - inject_datasetrolling_methods, has_bottleneck, bn) +from .common import full_like from .dask_array_ops import dask_rolling_wrapper +from .ops import ( + bn, has_bottleneck, inject_bottleneck_rolling_methods, + inject_datasetrolling_methods) +from .pycompat import OrderedDict, dask_array_type, zip class Rolling(object): diff --git a/xarray/core/utils.py b/xarray/core/utils.py index de6b5825390..25a60b87266 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -1,20 +1,19 @@ """Internal utilties; not for external use """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import contextlib import functools import itertools import re import warnings -from collections import Mapping, MutableMapping, MutableSet, Iterable +from collections import Iterable, Mapping, MutableMapping, MutableSet import numpy as np import pandas as pd -from .pycompat import (iteritems, OrderedDict, basestring, bytes_type, - dask_array_type) +from .pycompat import ( + OrderedDict, basestring, bytes_type, dask_array_type, iteritems) def alias_message(old_name, new_name): diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 267dc02ce13..efec2806f48 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1,29 +1,23 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from datetime import timedelta -from collections import defaultdict +from __future__ import absolute_import, division, print_function + import functools import itertools +from collections import defaultdict +from datetime import timedelta import numpy as np import pandas as pd -from . import common -from . import duck_array_ops -from . import dtypes -from . import indexing -from . import nputils -from . import ops -from . import utils -from .pycompat import (basestring, OrderedDict, zip, integer_types, - dask_array_type) -from .indexing import (PandasIndexAdapter, as_indexable, BasicIndexer, - OuterIndexer, VectorizedIndexer) -from .utils import OrderedSet - import xarray as xr # only for Dataset and DataArray +from . import common, dtypes, duck_array_ops, indexing, nputils, ops, utils +from .indexing import ( + BasicIndexer, OuterIndexer, PandasIndexAdapter, VectorizedIndexer, + as_indexable) +from .pycompat import ( + OrderedDict, basestring, dask_array_type, integer_types, zip) +from .utils import OrderedSet + try: import dask.array as da except ImportError: diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index badd44b25db..de715094834 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -1,18 +1,15 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -import warnings -import itertools import functools +import itertools +import warnings import numpy as np -from ..core.pycompat import getargspec from ..core.formatting import format_item -from .utils import (_determine_cmap_params, _infer_xy_labels, - import_matplotlib_pyplot) - +from ..core.pycompat import getargspec +from .utils import ( + _determine_cmap_params, _infer_xy_labels, import_matplotlib_pyplot) # Overrides axes.labelsize, xtick.major.size, ytick.major.size # from mpl.rcParams diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 57e3f101f4f..b5e6d94a4d2 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -5,21 +5,22 @@ Or use the methods on a DataArray: DataArray.plot._____ """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import functools import warnings +from datetime import datetime import numpy as np import pandas as pd -from datetime import datetime -from .utils import (ROBUST_PERCENTILE, _determine_cmap_params, - _infer_xy_labels, get_axis, import_matplotlib_pyplot) -from .facetgrid import FacetGrid from xarray.core.pycompat import basestring +from .facetgrid import FacetGrid +from .utils import ( + ROBUST_PERCENTILE, _determine_cmap_params, _infer_xy_labels, get_axis, + import_matplotlib_pyplot) + def _valid_numpy_subdtype(x, numpy_types): """ diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 0e565f24a60..497705302d2 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -1,16 +1,14 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import pkg_resources +from __future__ import absolute_import, division, print_function + import warnings import numpy as np import pandas as pd +import pkg_resources from ..core.pycompat import basestring from ..core.utils import is_scalar - ROBUST_PERCENTILE = 2.0 diff --git a/xarray/testing.py b/xarray/testing.py index 6b0a5b736de..ee5a54cd7dc 100644 --- a/xarray/testing.py +++ b/xarray/testing.py @@ -1,7 +1,5 @@ """Testing functions exposed to the user API""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np diff --git a/xarray/tests/test_accessors.py b/xarray/tests/test_accessors.py index 1fcde8f5a68..ad521546d2e 100644 --- a/xarray/tests/test_accessors.py +++ b/xarray/tests/test_accessors.py @@ -1,13 +1,12 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -import xarray as xr import numpy as np import pandas as pd -from . import (TestCase, requires_dask, raises_regex, assert_equal, - assert_array_equal) +import xarray as xr + +from . import ( + TestCase, assert_array_equal, assert_equal, raises_regex, requires_dask) class TestDatetimeAccessor(TestCase): diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index f82196212b0..32c79107a2c 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1,42 +1,40 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from io import BytesIO +from __future__ import absolute_import, division, print_function + import contextlib import itertools import os.path import pickle import shutil +import sys import tempfile import unittest -import sys import warnings +from io import BytesIO import numpy as np import pandas as pd import pytest import xarray as xr -from xarray import (Dataset, DataArray, open_dataset, open_dataarray, - open_mfdataset, backends, save_mfdataset) +from xarray import ( + DataArray, Dataset, backends, open_dataarray, open_dataset, open_mfdataset, + save_mfdataset) from xarray.backends.common import robust_getitem from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.backends.pydap_ import PydapDataStore from xarray.core import indexing -from xarray.core.pycompat import (iteritems, PY2, ExitStack, basestring, - dask_array_type) - -from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap, - requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf, - requires_pynio, requires_pathlib, requires_zarr, - requires_rasterio, has_netCDF4, has_scipy, assert_allclose, - flaky, network, assert_identical, raises_regex, assert_equal, - assert_array_equal) +from xarray.core.pycompat import ( + PY2, ExitStack, basestring, dask_array_type, iteritems) +from xarray.tests import mock +from . import ( + TestCase, assert_allclose, assert_array_equal, assert_equal, + assert_identical, flaky, has_netCDF4, has_scipy, network, raises_regex, + requires_dask, requires_h5netcdf, requires_netCDF4, requires_pathlib, + requires_pydap, requires_pynio, requires_rasterio, requires_scipy, + requires_scipy_or_netCDF4, requires_zarr) from .test_dataset import create_test_data -from xarray.tests import mock - try: import netCDF4 as nc4 except ImportError: diff --git a/xarray/tests/test_coding.py b/xarray/tests/test_coding.py index a6faea8749b..6300a1957f8 100644 --- a/xarray/tests/test_coding.py +++ b/xarray/tests/test_coding.py @@ -1,12 +1,11 @@ import numpy as np - import pytest import xarray as xr -from xarray.core.pycompat import suppress from xarray.coding import variables +from xarray.core.pycompat import suppress -from . import requires_dask, assert_identical +from . import assert_identical, requires_dask with suppress(ImportError): import dask.array as da diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 092559ce9da..b85f92ece66 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -1,17 +1,15 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import warnings import numpy as np import pandas as pd +import pytest from xarray import Variable, coding from xarray.coding.times import _import_netcdftime -from . import ( - TestCase, requires_netcdftime, assert_array_equal) -import pytest + +from . import TestCase, assert_array_equal, requires_netcdftime @np.vectorize diff --git a/xarray/tests/test_combine.py b/xarray/tests/test_combine.py index 365e274a191..09918d9a065 100644 --- a/xarray/tests/test_combine.py +++ b/xarray/tests/test_combine.py @@ -1,18 +1,17 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from copy import deepcopy +from __future__ import absolute_import, division, print_function -import pytest +from copy import deepcopy import numpy as np import pandas as pd +import pytest -from xarray import Dataset, DataArray, auto_combine, concat, Variable -from xarray.core.pycompat import iteritems, OrderedDict +from xarray import DataArray, Dataset, Variable, auto_combine, concat +from xarray.core.pycompat import OrderedDict, iteritems -from . import (TestCase, InaccessibleArray, requires_dask, raises_regex, - assert_equal, assert_identical, assert_array_equal) +from . import ( + InaccessibleArray, TestCase, assert_array_equal, assert_equal, + assert_identical, raises_regex, requires_dask) from .test_dataset import create_test_data diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index 23e77b83455..ebd51d04857 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -1,21 +1,20 @@ import functools import operator from collections import OrderedDict - from distutils.version import LooseVersion + import numpy as np -from numpy.testing import assert_array_equal import pandas as pd - import pytest +from numpy.testing import assert_array_equal import xarray as xr from xarray.core.computation import ( - _UFuncSignature, result_name, broadcast_compat_data, collect_dict_values, - join_dict_keys, ordered_set_intersection, ordered_set_union, - unified_dim_sizes, apply_ufunc) + _UFuncSignature, apply_ufunc, broadcast_compat_data, collect_dict_values, + join_dict_keys, ordered_set_intersection, ordered_set_union, result_name, + unified_dim_sizes) -from . import requires_dask, raises_regex +from . import raises_regex, requires_dask def assert_identical(a, b): diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 4520e7aefef..7028bac7057 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -1,26 +1,25 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import contextlib import warnings + import numpy as np import pandas as pd import pytest -from xarray import conventions, Variable, Dataset, open_dataset -from xarray.core import utils, indexing -from xarray.testing import assert_identical -from . import ( - TestCase, requires_netCDF4, requires_netcdftime, unittest, raises_regex, - IndexerMaker, assert_array_equal) -from .test_backends import CFEncodedDataTest -from xarray.core.pycompat import iteritems -from xarray.backends.memory import InMemoryDataStore +from xarray import Dataset, Variable, conventions, open_dataset from xarray.backends.common import WritableCFDataStore +from xarray.backends.memory import InMemoryDataStore from xarray.conventions import decode_cf +from xarray.core import indexing, utils +from xarray.core.pycompat import iteritems +from xarray.testing import assert_identical +from . import ( + IndexerMaker, TestCase, assert_array_equal, raises_regex, requires_netCDF4, + requires_netcdftime, unittest) +from .test_backends import CFEncodedDataTest B = IndexerMaker(indexing.BasicIndexer) V = IndexerMaker(indexing.VectorizedIndexer) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 7833a43a894..1e4f313897b 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1,28 +1,26 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import pickle +from distutils.version import LooseVersion from textwrap import dedent -from distutils.version import LooseVersion import numpy as np import pandas as pd import pytest import xarray as xr -from xarray import Variable, DataArray, Dataset import xarray.ufuncs as xu -from xarray.core.pycompat import suppress, OrderedDict -from . import ( - TestCase, assert_frame_equal, raises_regex, assert_equal, assert_identical, - assert_array_equal, assert_allclose) - +from xarray import DataArray, Dataset, Variable +from xarray.core.pycompat import OrderedDict, suppress from xarray.tests import mock +from . import ( + TestCase, assert_allclose, assert_array_equal, assert_equal, + assert_frame_equal, assert_identical, raises_regex) + dask = pytest.importorskip('dask') -import dask.array as da # noqa: E402 # allow importorskip call above this -import dask.dataframe as dd # noqa: E402 +da = pytest.importorskip('dask.array') +dd = pytest.importorskip('dask.dataframe') class DaskTestCase(TestCase): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index cd8a209d5ac..095ad5b793b 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1,25 +1,24 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import numpy as np -import pandas as pd +from __future__ import absolute_import, division, print_function + import pickle -import pytest from copy import deepcopy -from textwrap import dedent from distutils.version import LooseVersion +from textwrap import dedent -import xarray as xr +import numpy as np +import pandas as pd +import pytest -from xarray import (align, broadcast, Dataset, DataArray, - IndexVariable, Variable) +import xarray as xr +from xarray import ( + DataArray, Dataset, IndexVariable, Variable, align, broadcast) from xarray.coding.times import CFDatetimeCoder -from xarray.core.pycompat import iteritems, OrderedDict from xarray.core.common import full_like +from xarray.core.pycompat import OrderedDict, iteritems from xarray.tests import ( - TestCase, ReturnItem, source_ndarray, unittest, requires_dask, - assert_identical, assert_equal, assert_allclose, assert_array_equal, - raises_regex, requires_scipy, requires_bottleneck) + ReturnItem, TestCase, assert_allclose, assert_array_equal, assert_equal, + assert_identical, raises_regex, requires_bottleneck, requires_dask, + requires_scipy, source_ndarray, unittest) class TestDataArray(TestCase): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 4e746b90635..353128acd39 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1,36 +1,37 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + from copy import copy, deepcopy -from textwrap import dedent -try: - import cPickle as pickle -except ImportError: - import pickle -try: - import dask.array as da -except ImportError: - pass -from io import StringIO from distutils.version import LooseVersion +from io import StringIO +from textwrap import dedent import numpy as np import pandas as pd -import xarray as xr import pytest -from xarray import (align, broadcast, backends, Dataset, DataArray, Variable, - IndexVariable, open_dataset, set_options, MergeError) +import xarray as xr +from xarray import ( + DataArray, Dataset, IndexVariable, MergeError, Variable, align, backends, + broadcast, open_dataset, set_options) from xarray.core import indexing, utils -from xarray.core.pycompat import (iteritems, OrderedDict, unicode_type, - integer_types) from xarray.core.common import full_like +from xarray.core.pycompat import ( + OrderedDict, integer_types, iteritems, unicode_type) -from . import (TestCase, raises_regex, InaccessibleArray, UnexpectedDataAccess, - requires_dask, source_ndarray, assert_array_equal, assert_equal, - assert_allclose, assert_identical, requires_bottleneck, - requires_scipy) +from . import ( + InaccessibleArray, TestCase, UnexpectedDataAccess, assert_allclose, + assert_array_equal, assert_equal, assert_identical, raises_regex, + requires_bottleneck, requires_dask, requires_scipy, source_ndarray) + +try: + import cPickle as pickle +except ImportError: + import pickle +try: + import dask.array as da +except ImportError: + pass def create_test_data(seed=None): diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 47bb6cdc2e1..0d060069477 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -1,20 +1,29 @@ +""" isort:skip_file """ + import sys import pytest -import xarray as xr -distributed = pytest.importorskip('distributed') -da = pytest.importorskip('dask.array') -import dask +dask = pytest.importorskip('dask') # isort:skip +distributed = pytest.importorskip('distributed') # isort:skip + +from dask import array from distributed.utils_test import cluster, gen_cluster from distributed.utils_test import loop # flake8: noqa from distributed.client import futures_of -from xarray.tests.test_backends import create_tmp_file, ON_WINDOWS +import xarray as xr +from xarray.tests.test_backends import ON_WINDOWS, create_tmp_file from xarray.tests.test_dataset import create_test_data -from . import (assert_allclose, has_scipy, has_netCDF4, has_h5netcdf, - requires_zarr) +from . import ( + assert_allclose, has_h5netcdf, has_netCDF4, has_scipy, requires_zarr) + +# this is to stop isort throwing errors. May have been easier to just use +# `isort:skip` in retrospect + + +da = pytest.importorskip('dask.array') ENGINES = [] @@ -35,7 +44,8 @@ def test_dask_distributed_netcdf_integration_test(loop, engine): original = create_test_data() with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: original.to_netcdf(filename, engine=engine) - with xr.open_dataset(filename, chunks=3, engine=engine) as restored: + with xr.open_dataset( + filename, chunks=3, engine=engine) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed) diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 1b236e0160d..833df85f8af 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -1,6 +1,4 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np import pytest diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 5bb7e09d918..fde2a1cc726 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -1,20 +1,19 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import pytest +from __future__ import absolute_import, division, print_function + +from distutils.version import LooseVersion + import numpy as np +import pytest from numpy import array, nan -from distutils.version import LooseVersion -from . import assert_array_equal + +from xarray import DataArray, concat from xarray.core.duck_array_ops import ( - first, last, count, mean, array_notnull_equiv, where, stack, concatenate -) + array_notnull_equiv, concatenate, count, first, last, mean, stack, where) from xarray.core.pycompat import dask_array_type -from xarray import DataArray from xarray.testing import assert_allclose, assert_equal -from xarray import concat -from . import TestCase, raises_regex, has_dask, requires_dask +from . import ( + TestCase, assert_array_equal, has_dask, raises_regex, requires_dask) class TestOps(TestCase): diff --git a/xarray/tests/test_extensions.py b/xarray/tests/test_extensions.py index 9456f335572..24b710ae223 100644 --- a/xarray/tests/test_extensions.py +++ b/xarray/tests/test_extensions.py @@ -1,15 +1,15 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -try: - import cPickle as pickle -except ImportError: - import pickle +from __future__ import absolute_import, division, print_function + +import pytest import xarray as xr from . import TestCase, raises_regex -import pytest + +try: + import cPickle as pickle +except ImportError: + import pickle @xr.register_dataset_accessor('example_accessor') diff --git a/xarray/tests/test_formatting.py b/xarray/tests/test_formatting.py index 53342825dcd..34552891778 100644 --- a/xarray/tests/test_formatting.py +++ b/xarray/tests/test_formatting.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import numpy as np import pandas as pd diff --git a/xarray/tests/test_groupby.py b/xarray/tests/test_groupby.py index f1d80954295..fd53e410583 100644 --- a/xarray/tests/test_groupby.py +++ b/xarray/tests/test_groupby.py @@ -1,13 +1,12 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import numpy as np import pandas as pd +import pytest + import xarray as xr from xarray.core.groupby import _consolidate_slices -import pytest - def test_consolidate_slices(): diff --git a/xarray/tests/test_indexing.py b/xarray/tests/test_indexing.py index 4729aad9b79..4884eebe759 100644 --- a/xarray/tests/test_indexing.py +++ b/xarray/tests/test_indexing.py @@ -1,20 +1,17 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import itertools +from __future__ import absolute_import, division, print_function -import pytest +import itertools import numpy as np import pandas as pd +import pytest -from xarray import Dataset, DataArray, Variable -from xarray.core import indexing -from xarray.core import nputils +from xarray import DataArray, Dataset, Variable +from xarray.core import indexing, nputils from xarray.core.pycompat import native_int_types -from . import ( - TestCase, ReturnItem, raises_regex, IndexerMaker, assert_array_equal) +from . import ( + IndexerMaker, ReturnItem, TestCase, assert_array_equal, raises_regex) B = IndexerMaker(indexing.BasicIndexer) diff --git a/xarray/tests/test_merge.py b/xarray/tests/test_merge.py index 409ad86c1e9..4d89be8ce55 100644 --- a/xarray/tests/test_merge.py +++ b/xarray/tests/test_merge.py @@ -1,16 +1,14 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import numpy as np -import xarray as xr +from __future__ import absolute_import, division, print_function +import numpy as np import pytest +import xarray as xr +from xarray.core import merge + from . import TestCase, raises_regex from .test_dataset import create_test_data -from xarray.core import merge - class TestMergeInternals(TestCase): def test_broadcast_dimension_size(self): diff --git a/xarray/tests/test_missing.py b/xarray/tests/test_missing.py index ce735d720d0..1dde95adf42 100644 --- a/xarray/tests/test_missing.py +++ b/xarray/tests/test_missing.py @@ -1,20 +1,18 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + +import itertools + import numpy as np import pandas as pd import pytest -import itertools import xarray as xr - -from xarray.core.missing import (NumpyInterpolator, ScipyInterpolator, - SplineInterpolator) +from xarray.core.missing import ( + NumpyInterpolator, ScipyInterpolator, SplineInterpolator) from xarray.core.pycompat import dask_array_type - -from xarray.tests import (assert_equal, assert_array_equal, raises_regex, - requires_scipy, requires_bottleneck, requires_dask, - requires_np112) +from xarray.tests import ( + assert_array_equal, assert_equal, raises_regex, requires_bottleneck, + requires_dask, requires_np112, requires_scipy) @pytest.fixture diff --git a/xarray/tests/test_nputils.py b/xarray/tests/test_nputils.py index 83445e4639f..3c9c92ae2ba 100644 --- a/xarray/tests/test_nputils.py +++ b/xarray/tests/test_nputils.py @@ -1,7 +1,7 @@ import numpy as np from numpy.testing import assert_array_equal -from xarray.core.nputils import _is_contiguous, NumpyVIndexAdapter +from xarray.core.nputils import NumpyVIndexAdapter, _is_contiguous def test_is_contiguous(): diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index 498f0354086..aed96f1acb6 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -1,9 +1,8 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import xarray +from __future__ import absolute_import, division, print_function + import pytest +import xarray from xarray.core.options import OPTIONS diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 46410cd53e3..26ebcccc748 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1,30 +1,29 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# import mpl and change the backend before other mpl imports -try: - import matplotlib as mpl - import matplotlib.pyplot as plt -except ImportError: - pass +from __future__ import absolute_import, division, print_function import inspect +from datetime import datetime import numpy as np import pandas as pd -from datetime import datetime import pytest -from xarray import DataArray - import xarray.plot as xplt +from xarray import DataArray from xarray.plot.plot import _infer_interval_breaks -from xarray.plot.utils import (_determine_cmap_params, _build_discrete_cmap, - _color_palette, import_seaborn) +from xarray.plot.utils import ( + _build_discrete_cmap, _color_palette, _determine_cmap_params, + import_seaborn) + +from . import ( + TestCase, assert_array_equal, assert_equal, raises_regex, + requires_matplotlib, requires_seaborn) -from . import (TestCase, requires_matplotlib, requires_seaborn, raises_regex, - assert_equal, assert_array_equal) +# import mpl and change the backend before other mpl imports +try: + import matplotlib as mpl + import matplotlib.pyplot as plt +except ImportError: + pass @pytest.mark.flaky diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py index 02390ac277a..8a0fa5f6e48 100644 --- a/xarray/tests/test_testing.py +++ b/xarray/tests/test_testing.py @@ -1,6 +1,4 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import xarray as xr diff --git a/xarray/tests/test_tutorial.py b/xarray/tests/test_tutorial.py index 9ad797a9ac9..d550a85e8ce 100644 --- a/xarray/tests/test_tutorial.py +++ b/xarray/tests/test_tutorial.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -from xarray import tutorial, DataArray +from xarray import DataArray, tutorial from xarray.core.pycompat import suppress -from . import TestCase, network, assert_identical +from . import TestCase, assert_identical, network @network diff --git a/xarray/tests/test_ufuncs.py b/xarray/tests/test_ufuncs.py index 0d56285dfc1..64a246953fe 100644 --- a/xarray/tests/test_ufuncs.py +++ b/xarray/tests/test_ufuncs.py @@ -1,15 +1,13 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + import pickle import numpy as np -import xarray.ufuncs as xu import xarray as xr +import xarray.ufuncs as xu -from . import ( - TestCase, raises_regex, assert_identical, assert_array_equal) +from . import TestCase, assert_array_equal, assert_identical, raises_regex class TestOps(TestCase): diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 1a008eff180..3a76b6e8c92 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -1,14 +1,13 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import pytest +from __future__ import absolute_import, division, print_function import numpy as np import pandas as pd +import pytest from xarray.core import duck_array_ops, utils from xarray.core.pycompat import OrderedDict -from . import TestCase, requires_dask, assert_array_equal + +from . import TestCase, assert_array_equal, requires_dask class TestAlias(TestCase): diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index f5125796a77..5f60fc95e15 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1,35 +1,32 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + from collections import namedtuple from copy import copy, deepcopy from datetime import datetime, timedelta +from distutils.version import LooseVersion from textwrap import dedent -import pytest -from distutils.version import LooseVersion import numpy as np -import pytz import pandas as pd +import pytest +import pytz -from xarray import Variable, IndexVariable, Coordinate, Dataset +from xarray import Coordinate, Dataset, IndexVariable, Variable from xarray.core import indexing -from xarray.core.variable import as_variable, as_compatible_data -from xarray.core.indexing import (PandasIndexAdapter, LazilyIndexedArray, - BasicIndexer, OuterIndexer, - VectorizedIndexer, NumpyIndexingAdapter, - CopyOnWriteArray, MemoryCachedArray, - DaskIndexingAdapter) +from xarray.core.common import full_like, ones_like, zeros_like +from xarray.core.indexing import ( + BasicIndexer, CopyOnWriteArray, DaskIndexingAdapter, LazilyIndexedArray, + MemoryCachedArray, NumpyIndexingAdapter, OuterIndexer, PandasIndexAdapter, + VectorizedIndexer) from xarray.core.pycompat import PY3, OrderedDict -from xarray.core.common import full_like, zeros_like, ones_like from xarray.core.utils import NDArrayMixin +from xarray.core.variable import as_compatible_data, as_variable +from xarray.tests import requires_bottleneck from . import ( - TestCase, source_ndarray, requires_dask, raises_regex, assert_identical, - assert_array_equal, assert_equal, assert_allclose) - -from xarray.tests import requires_bottleneck + TestCase, assert_allclose, assert_array_equal, assert_equal, + assert_identical, raises_regex, requires_dask, source_ndarray) class VariableSubclassTestCases(object): diff --git a/xarray/tutorial.py b/xarray/tutorial.py index d7da63a328e..83a8317f42b 100644 --- a/xarray/tutorial.py +++ b/xarray/tutorial.py @@ -5,18 +5,14 @@ * building tutorials in the documentation. ''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import hashlib - import os as _os from .backends.api import open_dataset as _open_dataset from .core.pycompat import urlretrieve as _urlretrieve - _default_cache_dir = _os.sep.join(('~', '.xarray_tutorial_data')) diff --git a/xarray/ufuncs.py b/xarray/ufuncs.py index 1990ac5b765..f7f17aedc2b 100644 --- a/xarray/ufuncs.py +++ b/xarray/ufuncs.py @@ -13,20 +13,16 @@ Once NumPy 1.10 comes out with support for overriding ufuncs, this module will hopefully no longer be necessary. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as _np -from .core.variable import Variable as _Variable -from .core.dataset import Dataset as _Dataset from .core.dataarray import DataArray as _DataArray +from .core.dataset import Dataset as _Dataset +from .core.duck_array_ops import _dask_or_eager_func from .core.groupby import GroupBy as _GroupBy - from .core.pycompat import dask_array_type as _dask_array_type -from .core.duck_array_ops import _dask_or_eager_func - +from .core.variable import Variable as _Variable _xarray_types = (_Variable, _DataArray, _Dataset, _GroupBy) _dispatch_order = (_np.ndarray, _dask_array_type) + _xarray_types diff --git a/xarray/util/print_versions.py b/xarray/util/print_versions.py index b9bd6e88547..478b867b0af 100755 --- a/xarray/util/print_versions.py +++ b/xarray/util/print_versions.py @@ -2,14 +2,16 @@ see pandas/pandas/util/_print_versions.py''' +from __future__ import absolute_import + +import codecs +import importlib +import locale import os import platform -import sys import struct import subprocess -import codecs -import locale -import importlib +import sys def get_sys_info(): From ecf50d2131e30bc8fe33fbaf4af833f6c37ce0d9 Mon Sep 17 00:00:00 2001 From: Vijay Paul Date: Wed, 28 Feb 2018 10:19:57 +1300 Subject: [PATCH 039/282] DOC: Plot discrete colormap with proportional colorbar (#1930) * Plot discrete colormap with proportional colorbar Using 'cbar_kwargs' it is possible to extend the capabilities of colorbar ticks. * Fixing style errors. * Fixing style errors. * Fixed the line-length issue Shortened intro texts. * added absolute_import added absolute_import to make stickler happy, can be removed later. * Updated script with changes requested Renamed the script, script info updated. * Cosmetic changes --- doc/gallery/control_plot_colorbar.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 doc/gallery/control_plot_colorbar.py diff --git a/doc/gallery/control_plot_colorbar.py b/doc/gallery/control_plot_colorbar.py new file mode 100644 index 00000000000..a09d825f8f0 --- /dev/null +++ b/doc/gallery/control_plot_colorbar.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +=========================== +Control the plot's colorbar +=========================== + +Use ``cbar_kwargs`` keyword to specify the number of ticks. +The ``spacing`` kwarg can be used to draw proportional ticks. +""" +import xarray as xr +import matplotlib.pyplot as plt + +# Load the data +air_temp = xr.tutorial.load_dataset('air_temperature') +air2d = air_temp.air.isel(time=500) + +# Prepare the figure +f, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(14, 4)) + +# Irregular levels to illustrate the use of a proportional colorbar +levels = [245, 250, 255, 260, 265, 270, 275, 280, 285, 290, 310, 340] + +# Plot data +air2d.plot(ax=ax1, levels=levels) +air2d.plot(ax=ax2, levels=levels, cbar_kwargs={'ticks': levels}) +air2d.plot(ax=ax3, levels=levels, cbar_kwargs={'ticks': levels, + 'spacing': 'proportional'}) + +# Show plots +plt.tight_layout() +plt.show() From f3bbb3ef6badcfe5d1f3b77c231846f0e79a93ea Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Tue, 27 Feb 2018 18:21:40 -0500 Subject: [PATCH 040/282] DOC: Contributing / isort (#1947) * flake8 only without stepping on @jhamman's toes, flake8 takes 5.4s to run and so we can probably cut to just running `flake8` rather than an explanation of the diffs etc * isort instructions --- doc/contributing.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 26734e66ae6..acf9f8c94ec 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -345,18 +345,17 @@ the more common ``PEP8`` issues: :ref:`Continuous Integration ` will run the `flake8 `_ tool and report any stylistic errors in your code. Therefore, it is helpful before -submitting code to run the check yourself on the diff:: +submitting code to run the check yourself:: - git diff master -u -- "*.py" | flake8 --diff + flake8 -This command will catch any stylistic errors in your changes specifically, but -be beware it may not catch all of them. For example, if you delete the only -usage of an imported function, it is stylistically incorrect to import an -unused function. However, style-checking the diff will not catch this because -the actual import is not part of the diff. Thus, for completeness, you should -run this command, though it will take longer:: +If you install `isort `_ and +`flake8-isort `_, this will also show +any errors from incorrectly sorted imports. These aren't currently enforced in +CI. To automatically sort imports, you can run:: + + isort -y - flake8 xarray Backwards Compatibility ~~~~~~~~~~~~~~~~~~~~~~~ From dc3eebf3a514cfdc1039b63f2a542121d1328ba9 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Thu, 1 Mar 2018 12:39:18 +0900 Subject: [PATCH 041/282] Rolling window with `as_strided` (#1837) * Rolling_window for np.ndarray * Add pad method to Variable * Added rolling_window to DataArray and Dataset * remove pad_value option. Support dask.rolling_window * Refactor rolling.reduce * add as_strided to npcompat. Tests added for reduce(np.nanmean) * Support boolean in maybe_promote * move rolling_window into duck_array_op. Make DataArray.rolling_window public. * Added to_dataarray and to_dataset to rolling object. * Use pad in rolling to make compatible to pandas. Expose pad_with_fill_value to public. * Refactor rolling * flake8 * Added a comment for dask's pad. * Use fastpath in rolling.to_dataarray * Doc added. * Revert not to use fastpath * Remove maybe_prompt for Boolean. Some improvements based on @shoyer's review. * Update test. * Bug fix in test_rolling_count_correct * fill_value for boolean array * rolling_window(array, axis, window) -> rolling_window(array, window, axis) * support stride in rolling.to_dataarray * flake8 * Improve doc. Add DataArrayRolling to api.rst * Improve docs in common.rolling. * Expose groupby docs to public * Default fill_value=dtypes.NA, stride=1. Add comment for DataArrayRollig. * Default fill_value=dtypes.NA, stride=1. Add comment for DataArrayRollig. * Add fill_value option to rolling.to_dataarray * Convert non-numeric array in reduce. * Fill_value = False for boolean array in rolling.reduce * Support old numpy plus bottleneck combination. Suppress warning for all-nan slice reduce. * flake8 * Add benchmark * Dataset.count. Benchmark * Classize benchmark * Decoratorize for asv benchmark * Classize benchmarks/indexing.py * Working with nanreduce * Support .sum for object dtype. * Remove unused if-statements. * Default skipna for rolling.reduce * Pass tests. Test added to make sure the consistency to pandas' behavior. * Delete duplicate file. flake8 * flake8 again * Working with numpy<1.13 * Revert "Classize benchmarks/indexing.py" This reverts commit 4189d71d9998ff83deb9a5d7035a2edaf628ae25. * rolling_window with dask.ghost * Optimize rolling.count. * Fixing style errors. * Remove unused npcompat.nansum etc * flake8 * require_dask -> has_dask * npcompat -> np * flake8 * Skip tests for old numpy. * Improve doc. Optmize missing._get_valid_fill_mask * to_dataarray -> construct * remove assert_allclose_with_nan * Fixing style errors. * typo * `to_dataset` -> `construct` * Update doc * Change boundary and add comments for dask_rolling_window. * Refactor dask_array_ops.rolling_window and np_utils.rolling_window * flake8 * Simplify tests * flake8 again. * cleanup roling_window for dask. * remove duplicates * remvove duplicate * flake8 * delete unnecessary file. --- asv_bench/benchmarks/__init__.py | 8 + asv_bench/benchmarks/rolling.py | 50 +++++ doc/api.rst | 26 +++ doc/computation.rst | 30 ++- doc/whats-new.rst | 14 +- xarray/core/common.py | 16 +- xarray/core/dask_array_ops.py | 74 ++++++- xarray/core/duck_array_ops.py | 19 +- xarray/core/missing.py | 7 +- xarray/core/npcompat.py | 11 ++ xarray/core/nputils.py | 64 ++++++ xarray/core/ops.py | 16 +- xarray/core/rolling.py | 290 ++++++++++++++++++---------- xarray/core/variable.py | 98 ++++++++++ xarray/tests/test_dataarray.py | 113 ++++++++--- xarray/tests/test_dataset.py | 36 +++- xarray/tests/test_duck_array_ops.py | 28 ++- xarray/tests/test_nputils.py | 27 ++- xarray/tests/test_variable.py | 56 +++++- 19 files changed, 824 insertions(+), 159 deletions(-) create mode 100644 asv_bench/benchmarks/rolling.py diff --git a/asv_bench/benchmarks/__init__.py b/asv_bench/benchmarks/__init__.py index f9bbc751284..997fdfd0db0 100644 --- a/asv_bench/benchmarks/__init__.py +++ b/asv_bench/benchmarks/__init__.py @@ -8,6 +8,14 @@ _counter = itertools.count() +def parameterized(names, params): + def decorator(func): + func.param_names = names + func.params = params + return func + return decorator + + def requires_dask(): try: import dask # noqa diff --git a/asv_bench/benchmarks/rolling.py b/asv_bench/benchmarks/rolling.py new file mode 100644 index 00000000000..79d06019c00 --- /dev/null +++ b/asv_bench/benchmarks/rolling.py @@ -0,0 +1,50 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import pandas as pd +import xarray as xr + +from . import parameterized, randn, requires_dask + +nx = 3000 +ny = 2000 +nt = 1000 +window = 20 + + +class Rolling(object): + def setup(self, *args, **kwargs): + self.ds = xr.Dataset( + {'var1': (('x', 'y'), randn((nx, ny), frac_nan=0.1)), + 'var2': (('x', 't'), randn((nx, nt))), + 'var3': (('t', ), randn(nt))}, + coords={'x': np.arange(nx), + 'y': np.linspace(0, 1, ny), + 't': pd.date_range('1970-01-01', periods=nt, freq='D'), + 'x_coords': ('x', np.linspace(1.1, 2.1, nx))}) + + @parameterized(['func', 'center'], + (['mean', 'count'], [True, False])) + def time_rolling(self, func, center): + getattr(self.ds.rolling(x=window, center=center), func)() + + @parameterized(['window_', 'min_periods'], + ([20, 40], [5, None])) + def time_rolling_np(self, window_, min_periods): + self.ds.rolling(x=window_, center=False, + min_periods=min_periods).reduce(getattr(np, 'nanmean')) + + @parameterized(['center', 'stride'], + ([True, False], [1, 200])) + def time_rolling_construct(self, center, stride): + self.ds.rolling(x=window, center=center).construct( + 'window_dim', stride=stride).mean(dim='window_dim') + + +class RollingDask(Rolling): + def setup(self, *args, **kwargs): + requires_dask() + super(RollingDask, self).setup(**kwargs) + self.ds = self.ds.chunk({'x': 100, 'y': 50, 't': 50}) diff --git a/doc/api.rst b/doc/api.rst index 10386fe3a9b..4a26298b268 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -467,6 +467,32 @@ DataArray methods DataArray.load DataArray.chunk +Rolling objects +=============== + +.. autosummary:: + :toctree: generated/ + + core.rolling.DataArrayRolling + core.rolling.DataArrayRolling.construct + core.rolling.DataArrayRolling.reduce + core.rolling.DatasetRolling + core.rolling.DatasetRolling.construct + core.rolling.DatasetRolling.reduce + +GroupByObjects +============== + +.. autosummary:: + :toctree: generated/ + + core.groupby.DataArrayGroupBy + core.groupby.DataArrayGroupBy.apply + core.groupby.DataArrayGroupBy.reduce + core.groupby.DatasetGroupBy + core.groupby.DatasetGroupBy.apply + core.groupby.DatasetGroupBy.reduce + Plotting ======== diff --git a/doc/computation.rst b/doc/computation.rst index 420b97923d7..78c645ff8c3 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -158,13 +158,11 @@ Aggregation and summary methods can be applied directly to the ``Rolling`` objec r.mean() r.reduce(np.std) -Note that rolling window aggregations are much faster (both asymptotically and -because they avoid a loop in Python) when bottleneck_ is installed. Otherwise, -we fall back to a slower, pure Python implementation. +Note that rolling window aggregations are faster when bottleneck_ is installed. .. _bottleneck: https://github.com/kwgoodman/bottleneck/ -Finally, we can manually iterate through ``Rolling`` objects: +We can also manually iterate through ``Rolling`` objects: .. ipython:: python @@ -172,6 +170,30 @@ Finally, we can manually iterate through ``Rolling`` objects: for label, arr_window in r: # arr_window is a view of x +Finally, the rolling object has ``construct`` method, which gives a +view of the original ``DataArray`` with the windowed dimension attached to +the last position. +You can use this for more advanced rolling operations, such as strided rolling, +windowed rolling, convolution, short-time FFT, etc. + +.. ipython:: python + + # rolling with 2-point stride + rolling_da = r.construct('window_dim', stride=2) + rolling_da + rolling_da.mean('window_dim', skipna=False) + +Because the ``DataArray`` given by ``r.construct('window_dim')`` is a view +of the original array, it is memory efficient. + +.. note:: + numpy's Nan-aggregation functions such as ``nansum`` copy the original array. + In xarray, we internally use these functions in our aggregation methods + (such as ``.sum()``) if ``skipna`` argument is not specified or set to True. + This means ``rolling_da.mean('window_dim')`` is memory inefficient. + To avoid this, use ``skipna=False`` as the above example. + + .. _compute.broadcasting: Broadcasting by dimension name diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8b200103303..ab667ceba3f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -38,6 +38,15 @@ Documentation Enhancements ~~~~~~~~~~~~ +- Improve :py:func:`~xarray.DataArray.rolling` logic. + :py:func:`~xarray.DataArrayRolling` object now supports + :py:func:`~xarray.DataArrayRolling.construct` method that returns a view + of the DataArray / Dataset object with the rolling-window dimension added + to the last axis. This enables more flexible operation, such as strided + rolling, windowed rolling, ND-rolling, short-time FFT and convolution. + (:issue:`1831`, :issue:`1142`, :issue:`819`) + By `Keisuke Fujii `_. + Bug fixes ~~~~~~~~~ @@ -64,7 +73,6 @@ Documentation Enhancements ~~~~~~~~~~~~ - **New functions and methods**: - Added :py:meth:`DataArray.to_iris` and @@ -140,6 +148,10 @@ Enhancements Bug fixes ~~~~~~~~~ +- Rolling aggregation with ``center=True`` option now gives the same result + with pandas including the last element (:issue:`1046`). + By `Keisuke Fujii `_. + - Support indexing with a 0d-np.ndarray (:issue:`1921`). By `Keisuke Fujii `_. - Added warning in api.py of a netCDF4 bug that occurs when diff --git a/xarray/core/common.py b/xarray/core/common.py index d521e7ae5c2..85ac0bf9364 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -424,6 +424,11 @@ def groupby(self, group, squeeze=True): grouped : GroupBy A `GroupBy` object patterned after `pandas.GroupBy` that can be iterated over in the form of `(unique_value, grouped_array)` pairs. + + See Also + -------- + core.groupby.DataArrayGroupBy + core.groupby.DatasetGroupBy """ return self._groupby_cls(self, group, squeeze=squeeze) @@ -483,9 +488,6 @@ def rolling(self, min_periods=None, center=False, **windows): """ Rolling window object. - Rolling window aggregations are much faster when bottleneck is - installed. - Parameters ---------- min_periods : int, default None @@ -503,7 +505,8 @@ def rolling(self, min_periods=None, center=False, **windows): Returns ------- - rolling : type of input argument + Rolling object (core.rolling.DataArrayRolling for DataArray, + core.rolling.DatasetRolling for Dataset.) Examples -------- @@ -531,6 +534,11 @@ def rolling(self, min_periods=None, center=False, **windows): array([ 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.]) Coordinates: * time (time) datetime64[ns] 2000-02-15 2000-03-15 2000-04-15 ... + + See Also + -------- + core.rolling.DataArrayRolling + core.rolling.DatasetRolling """ return self._rolling_cls(self, min_periods=min_periods, diff --git a/xarray/core/dask_array_ops.py b/xarray/core/dask_array_ops.py index 3aefd114517..5524efb4803 100644 --- a/xarray/core/dask_array_ops.py +++ b/xarray/core/dask_array_ops.py @@ -1,6 +1,9 @@ -"""Define core operations for xarray objects. -""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np +from . import nputils try: import dask.array as da @@ -24,3 +27,70 @@ def dask_rolling_wrapper(moving_func, a, window, min_count=None, axis=-1): # trim array result = da.ghost.trim_internal(out, depth) return result + + +def rolling_window(a, axis, window, center, fill_value): + """ Dask's equivalence to np.utils.rolling_window """ + orig_shape = a.shape + # inputs for ghost + if axis < 0: + axis = a.ndim + axis + depth = {d: 0 for d in range(a.ndim)} + depth[axis] = int(window / 2) + # For evenly sized window, we need to crop the first point of each block. + offset = 1 if window % 2 == 0 else 0 + + if depth[axis] > min(a.chunks[axis]): + raise ValueError( + "For window size %d, every chunk should be larger than %d, " + "but the smallest chunk size is %d. Rechunk your array\n" + "with a larger chunk size or a chunk size that\n" + "more evenly divides the shape of your array." % + (window, depth[axis], min(a.chunks[axis]))) + + # Although dask.ghost pads values to boundaries of the array, + # the size of the generated array is smaller than what we want + # if center == False. + if center: + start = int(window / 2) # 10 -> 5, 9 -> 4 + end = window - 1 - start + else: + start, end = window - 1, 0 + pad_size = max(start, end) + offset - depth[axis] + drop_size = 0 + # pad_size becomes more than 0 when the ghosted array is smaller than + # needed. In this case, we need to enlarge the original array by padding + # before ghosting. + if pad_size > 0: + if pad_size < depth[axis]: + # Ghosting requires each chunk larger than depth. If pad_size is + # smaller than the depth, we enlarge this and truncate it later. + drop_size = depth[axis] - pad_size + pad_size = depth[axis] + shape = list(a.shape) + shape[axis] = pad_size + chunks = list(a.chunks) + chunks[axis] = (pad_size, ) + fill_array = da.full(shape, fill_value, dtype=a.dtype, chunks=chunks) + a = da.concatenate([fill_array, a], axis=axis) + + boundary = {d: fill_value for d in range(a.ndim)} + + # create ghosted arrays + ag = da.ghost.ghost(a, depth=depth, boundary=boundary) + + # apply rolling func + def func(x, window, axis=-1): + x = np.asarray(x) + rolling = nputils._rolling_window(x, window, axis) + return rolling[(slice(None), ) * axis + (slice(offset, None), )] + + chunks = list(a.chunks) + chunks.append(window) + out = ag.map_blocks(func, dtype=a.dtype, new_axis=a.ndim, chunks=chunks, + window=window, axis=axis) + + # crop boundary. + index = (slice(None),) * axis + (slice(drop_size, + drop_size + orig_shape[axis]), ) + return out[index] diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 1a1bcf36c56..3a5c4a124d1 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -13,7 +13,7 @@ import numpy as np import pandas as pd -from . import dtypes, npcompat +from . import dask_array_ops, dtypes, npcompat, nputils from .nputils import nanfirst, nanlast from .pycompat import dask_array_type @@ -278,6 +278,10 @@ def f(values, axis=None, skipna=None, **kwargs): dtype = kwargs.get('dtype', None) values = asarray(values) + # dask requires dtype argument for object dtype + if (values.dtype == 'object' and name in ['sum', ]): + kwargs['dtype'] = values.dtype if dtype is None else dtype + if coerce_strings and values.dtype.kind in 'SU': values = values.astype(object) @@ -369,3 +373,16 @@ def last(values, axis, skipna=None): _fail_on_dask_array_input_skipna(values) return nanlast(values, axis) return take(values, -1, axis=axis) + + +def rolling_window(array, axis, window, center, fill_value): + """ + Make an ndarray with a rolling window of axis-th dimension. + The rolling dimension will be placed at the last dimension. + """ + if isinstance(array, dask_array_type): + return dask_array_ops.rolling_window( + array, axis, window, center, fill_value) + else: # np.ndarray + return nputils.rolling_window( + array, axis, window, center, fill_value) diff --git a/xarray/core/missing.py b/xarray/core/missing.py index e58d74f4c0d..0da6750f5bc 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from . import rolling from .computation import apply_ufunc from .npcompat import flip from .pycompat import iteritems @@ -326,4 +327,8 @@ def _get_valid_fill_mask(arr, dim, limit): '''helper function to determine values that can be filled when limit is not None''' kw = {dim: limit + 1} - return arr.isnull().rolling(min_periods=1, **kw).sum() <= limit + # we explicitly use construct method to avoid copy. + new_dim = rolling._get_new_dimname(arr.dims, '_window') + return (arr.isnull().rolling(min_periods=1, **kw) + .construct(new_dim, fill_value=False) + .sum(new_dim, skipna=False)) <= limit diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index df1e955518c..8f1f3821f96 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -1,6 +1,17 @@ from __future__ import absolute_import, division, print_function import numpy as np +from distutils.version import LooseVersion + + +if LooseVersion(np.__version__) >= LooseVersion('1.12'): + as_strided = np.lib.stride_tricks.as_strided +else: + def as_strided(x, shape=None, strides=None, subok=False, writeable=True): + array = np.lib.stride_tricks.as_strided(x, shape, strides, subok) + array.setflags(write=writeable) + return array + try: from numpy import nancumsum, nancumprod, flip diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index c781ca65a69..4ca1f9390eb 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -5,6 +5,8 @@ import numpy as np import pandas as pd +from . import npcompat + def _validate_axis(data, axis): ndim = data.ndim @@ -133,3 +135,65 @@ def __setitem__(self, key, value): mixed_positions, vindex_positions = _advanced_indexer_subspaces(key) self._array[key] = np.moveaxis(value, vindex_positions, mixed_positions) + + +def rolling_window(a, axis, window, center, fill_value): + """ rolling window with padding. """ + pads = [(0, 0) for s in a.shape] + if center: + start = int(window / 2) # 10 -> 5, 9 -> 4 + end = window - 1 - start + pads[axis] = (start, end) + else: + pads[axis] = (window - 1, 0) + a = np.pad(a, pads, mode='constant', constant_values=fill_value) + return _rolling_window(a, window, axis) + + +def _rolling_window(a, window, axis=-1): + """ + Make an ndarray with a rolling window along axis. + + Parameters + ---------- + a : array_like + Array to add rolling window to + axis: int + axis position along which rolling window will be applied. + window : int + Size of rolling window + + Returns + ------- + Array that is a view of the original array with a added dimension + of size w. + + Examples + -------- + >>> x=np.arange(10).reshape((2,5)) + >>> np.rolling_window(x, 3, axis=-1) + array([[[0, 1, 2], [1, 2, 3], [2, 3, 4]], + [[5, 6, 7], [6, 7, 8], [7, 8, 9]]]) + + Calculate rolling mean of last dimension: + >>> np.mean(np.rolling_window(x, 3, axis=-1), -1) + array([[ 1., 2., 3.], + [ 6., 7., 8.]]) + + This function is taken from https://github.com/numpy/numpy/pull/31 + but slightly modified to accept axis option. + """ + axis = _validate_axis(a, axis) + a = np.swapaxes(a, axis, -1) + + if window < 1: + raise ValueError( + "`window` must be at least 1. Given : {}".format(window)) + if window > a.shape[-1]: + raise ValueError("`window` is too long. Given : {}".format(window)) + + shape = a.shape[:-1] + (a.shape[-1] - window + 1, window) + strides = a.strides + (a.strides[-1],) + rolling = npcompat.as_strided(a, shape=shape, strides=strides, + writeable=False) + return np.swapaxes(rolling, -2, axis) diff --git a/xarray/core/ops.py b/xarray/core/ops.py index 32b31010b5f..d9e8ceb65d5 100644 --- a/xarray/core/ops.py +++ b/xarray/core/ops.py @@ -223,20 +223,8 @@ def func(self, *args, **kwargs): def rolling_count(rolling): - not_null = rolling.obj.notnull() - instance_attr_dict = {'center': rolling.center, - 'min_periods': rolling.min_periods, - rolling.dim: rolling.window} - rolling_count = not_null.rolling(**instance_attr_dict).sum() - - if rolling.min_periods is None: - return rolling_count - - # otherwise we need to filter out points where there aren't enough periods - # but not_null is False, and so the NaNs don't flow through - # array with points where there are enough values given min_periods - enough_periods = rolling_count >= rolling.min_periods - + rolling_count = rolling._counts() + enough_periods = rolling_count >= rolling._min_periods return rolling_count.where(enough_periods) diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 4bb020cebeb..845dcae5473 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -5,8 +5,7 @@ import numpy as np -from .combine import concat -from .common import full_like +from . import dtypes from .dask_array_ops import dask_rolling_wrapper from .ops import ( bn, has_bottleneck, inject_bottleneck_rolling_methods, @@ -14,6 +13,24 @@ from .pycompat import OrderedDict, dask_array_type, zip +def _get_new_dimname(dims, new_dim): + """ Get an new dimension name based on new_dim, that is not used in dims. + If the same name exists, we add an underscore(s) in the head. + + Example1: + dims: ['a', 'b', 'c'] + new_dim: ['_rolling'] + -> ['_rolling'] + Example2: + dims: ['a', 'b', 'c', '_rolling'] + new_dim: ['_rolling'] + -> ['__rolling'] + """ + while new_dim in dims: + new_dim = '_' + new_dim + return new_dim + + class Rolling(object): """A object that implements the moving window pattern. @@ -98,59 +115,53 @@ def __len__(self): class DataArrayRolling(Rolling): - """ - This class adds the following class methods; - + _reduce_method(cls, func) - + _bottleneck_reduce(cls, func) - - These class methods will be used to inject numpy or bottleneck function - by doing - - >>> func = cls._reduce_method(f) - >>> func.__name__ = name - >>> setattr(cls, name, func) - - in ops.inject_bottleneck_rolling_methods. - - After the injection, the Rolling object will have `name` (such as `mean` or - `median`) methods, - e.g. it enables the following call, - >>> data.rolling().mean() + def __init__(self, obj, min_periods=None, center=False, **windows): + """ + Moving window object for DataArray. + You should use DataArray.rolling() method to construct this object + instead of the class constructor. - If bottleneck is installed, some bottleneck methods will be used instdad of - the numpy method. + Parameters + ---------- + obj : DataArray + Object to window. + min_periods : int, default None + Minimum number of observations in window required to have a value + (otherwise result is NA). The default, None, is equivalent to + setting min_periods equal to the size of the window. + center : boolean, default False + Set the labels at the center of the window. + **windows : dim=window + dim : str + Name of the dimension to create the rolling iterator + along (e.g., `time`). + window : int + Size of the moving window. - see also - + rolling.DataArrayRolling - + ops.inject_bottleneck_rolling_methods - """ + Returns + ------- + rolling : type of input argument - def __init__(self, obj, min_periods=None, center=False, **windows): + See Also + -------- + DataArray.rolling + DataArray.groupby + Dataset.rolling + Dataset.groupby + """ super(DataArrayRolling, self).__init__(obj, min_periods=min_periods, center=center, **windows) - self._windows = None - self._valid_windows = None self.window_indices = None self.window_labels = None self._setup_windows() - @property - def windows(self): - if self._windows is None: - self._windows = OrderedDict(zip(self.window_labels, - self.window_indices)) - return self._windows - def __iter__(self): - for (label, indices, valid) in zip(self.window_labels, - self.window_indices, - self._valid_windows): - + for (label, indices) in zip(self.window_labels, self.window_indices): window = self.obj.isel(**{self.dim: indices}) - if not valid: - window = full_like(window, fill_value=True, dtype=bool) + counts = window.count(dim=self.dim) + window = window.where(counts >= self._min_periods) yield (label, window) @@ -158,32 +169,65 @@ def _setup_windows(self): """ Find the indices and labels for each window """ - from .dataarray import DataArray - self.window_labels = self.obj[self.dim] - window = int(self.window) - dim_size = self.obj[self.dim].size stops = np.arange(dim_size) + 1 starts = np.maximum(stops - window, 0) - if self._min_periods > 1: - valid_windows = (stops - starts) >= self._min_periods - else: - # No invalid windows - valid_windows = np.ones(dim_size, dtype=bool) - self._valid_windows = DataArray(valid_windows, dims=(self.dim, ), - coords=self.obj[self.dim].coords) - self.window_indices = [slice(start, stop) for start, stop in zip(starts, stops)] - def _center_result(self, result): - """center result""" - shift = (-self.window // 2) + 1 - return result.shift(**{self.dim: shift}) + def construct(self, window_dim, stride=1, fill_value=dtypes.NA): + """ + Convert this rolling object to xr.DataArray, + where the window dimension is stacked as a new dimension + + Parameters + ---------- + window_dim: str + New name of the window dimension. + stride: integer, optional + Size of stride for the rolling window. + fill_value: optional. Default dtypes.NA + Filling value to match the dimension size. + + Returns + ------- + DataArray that is a view of the original array. + + Note + ---- + The return array is not writeable. + + Examples + -------- + >>> da = DataArray(np.arange(8).reshape(2, 4), dims=('a', 'b')) + >>> + >>> rolling = da.rolling(a=3) + >>> rolling.to_datarray('window_dim') + + array([[[np.nan, np.nan, 0], [np.nan, 0, 1], [0, 1, 2], [1, 2, 3]], + [[np.nan, np.nan, 4], [np.nan, 4, 5], [4, 5, 6], [5, 6, 7]]]) + Dimensions without coordinates: a, b, window_dim + >>> + >>> rolling = da.rolling(a=3, center=True) + >>> rolling.to_datarray('window_dim') + + array([[[np.nan, 0, 1], [0, 1, 2], [1, 2, 3], [2, 3, np.nan]], + [[np.nan, 4, 5], [4, 5, 6], [5, 6, 7], [6, 7, np.nan]]]) + Dimensions without coordinates: a, b, window_dim + """ + + from .dataarray import DataArray + + window = self.obj.variable.rolling_window(self.dim, self.window, + window_dim, self.center, + fill_value=fill_value) + result = DataArray(window, dims=self.obj.dims + (window_dim,), + coords=self.obj.coords) + return result.isel(**{self.dim: slice(None, None, stride)}) def reduce(self, func, **kwargs): """Reduce the items in this group by applying `func` along some @@ -203,27 +247,27 @@ def reduce(self, func, **kwargs): reduced : DataArray Array with summarized data. """ - - windows = [window.reduce(func, dim=self.dim, **kwargs) - for _, window in self] - - # Find valid windows based on count - if self.dim in self.obj.coords: - concat_dim = self.window_labels - else: - concat_dim = self.dim - counts = concat([window.count(dim=self.dim) for _, window in self], - dim=concat_dim) - result = concat(windows, dim=concat_dim) - # restore dim order - result = result.transpose(*self.obj.dims) - - result = result.where(counts >= self._min_periods) - - if self.center: - result = self._center_result(result) - - return result + rolling_dim = _get_new_dimname(self.obj.dims, '_rolling_dim') + windows = self.construct(rolling_dim) + result = windows.reduce(func, dim=rolling_dim, **kwargs) + + # Find valid windows based on count. + counts = self._counts() + return result.where(counts >= self._min_periods) + + def _counts(self): + """ Number of non-nan entries in each rolling window. """ + + rolling_dim = _get_new_dimname(self.obj.dims, '_rolling_dim') + # We use False as the fill_value instead of np.nan, since boolean + # array is faster to be reduced than object array. + # The use of skipna==False is also faster since it does not need to + # copy the strided array. + counts = (self.obj.notnull() + .rolling(center=self.center, **{self.dim: self.window}) + .construct(rolling_dim, fill_value=False) + .sum(dim=rolling_dim, skipna=False)) + return counts @classmethod def _reduce_method(cls, func): @@ -255,45 +299,41 @@ def wrapped_func(self, **kwargs): axis = self.obj.get_axis_num(self.dim) - if isinstance(self.obj.data, dask_array_type): + padded = self.obj.variable + if self.center: + shift = (-self.window // 2) + 1 + + if (LooseVersion(np.__version__) < LooseVersion('1.13') and + self.obj.dtype.kind == 'b'): + # with numpy < 1.13 bottleneck cannot handle np.nan-Boolean + # mixed array correctly. We cast boolean array to float. + padded = padded.astype(float) + padded = padded.pad_with_fill_value(**{self.dim: (0, -shift)}) + valid = (slice(None), ) * axis + (slice(-shift, None), ) + + if isinstance(padded.data, dask_array_type): values = dask_rolling_wrapper(func, self.obj.data, window=self.window, min_count=min_count, axis=axis) else: - values = func(self.obj.data, window=self.window, + values = func(padded.data, window=self.window, min_count=min_count, axis=axis) - result = DataArray(values, self.obj.coords) - if self.center: - result = self._center_result(result) + values = values[valid] + result = DataArray(values, self.obj.coords) return result return wrapped_func class DatasetRolling(Rolling): - """An object that implements the moving window pattern for Dataset. - - This class has an OrderedDict named self.rollings, that is a collection of - DataArrayRollings for all the DataArrays in the Dataset, except for those - not depending on rolling dimension. - - reduce() method returns a new Dataset generated from a set of - self.rollings[key].reduce(). - - See Also - -------- - Dataset.groupby - DataArray.groupby - Dataset.rolling - DataArray.rolling - """ - def __init__(self, obj, min_periods=None, center=False, **windows): """ Moving window object for Dataset. + You should use Dataset.rolling() method to construct this object + instead of the class constructor. Parameters ---------- @@ -315,6 +355,13 @@ def __init__(self, obj, min_periods=None, center=False, **windows): Returns ------- rolling : type of input argument + + See Also + -------- + Dataset.rolling + DataArray.rolling + Dataset.groupby + DataArray.groupby """ super(DatasetRolling, self).__init__(obj, min_periods, center, **windows) @@ -355,6 +402,16 @@ def reduce(self, func, **kwargs): reduced[key] = self.obj[key] return Dataset(reduced, coords=self.obj.coords) + def _counts(self): + from .dataset import Dataset + reduced = OrderedDict() + for key, da in self.obj.data_vars.items(): + if self.dim in da.dims: + reduced[key] = self.rollings[key]._counts() + else: + reduced[key] = self.obj[key] + return Dataset(reduced, coords=self.obj.coords) + @classmethod def _reduce_method(cls, func): """ @@ -374,6 +431,37 @@ def wrapped_func(self, **kwargs): return Dataset(reduced, coords=self.obj.coords) return wrapped_func + def construct(self, window_dim, stride=1, fill_value=dtypes.NA): + """ + Convert this rolling object to xr.Dataset, + where the window dimension is stacked as a new dimension + + Parameters + ---------- + window_dim: str + New name of the window dimension. + stride: integer, optional + size of stride for the rolling window. + fill_value: optional. Default dtypes.NA + Filling value to match the dimension size. + + Returns + ------- + Dataset with variables converted from rolling object. + """ + + from .dataset import Dataset + + dataset = OrderedDict() + for key, da in self.obj.data_vars.items(): + if self.dim in da.dims: + dataset[key] = self.rollings[key].construct( + window_dim, fill_value=fill_value) + else: + dataset[key] = da + return Dataset(dataset, coords=self.obj.coords).isel( + **{self.dim: slice(None, None, stride)}) + inject_bottleneck_rolling_methods(DataArrayRolling) inject_datasetrolling_methods(DatasetRolling) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index efec2806f48..bb4285fba0a 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -934,6 +934,53 @@ def shift(self, **shifts): result = result._shift_one_dim(dim, count) return result + def pad_with_fill_value(self, fill_value=dtypes.NA, **pad_widths): + """ + Return a new Variable with paddings. + + Parameters + ---------- + **pad_width: keyword arguments of the form {dim: (before, after)} + Number of values padded to the edges of each dimension. + """ + if fill_value is dtypes.NA: # np.nan is passed + dtype, fill_value = dtypes.maybe_promote(self.dtype) + else: + dtype = self.dtype + + if isinstance(self.data, dask_array_type): + array = self.data + + # Dask does not yet support pad. We manually implement it. + # https://github.com/dask/dask/issues/1926 + for d, pad in pad_widths.items(): + axis = self.get_axis_num(d) + before_shape = list(array.shape) + before_shape[axis] = pad[0] + before_chunks = list(array.chunks) + before_chunks[axis] = (pad[0], ) + after_shape = list(array.shape) + after_shape[axis] = pad[1] + after_chunks = list(array.chunks) + after_chunks[axis] = (pad[1], ) + + arrays = [] + if pad[0] > 0: + arrays.append(da.full(before_shape, fill_value, + dtype=dtype, chunks=before_chunks)) + arrays.append(array) + if pad[1] > 0: + arrays.append(da.full(after_shape, fill_value, + dtype=dtype, chunks=after_chunks)) + if len(arrays) > 1: + array = da.concatenate(arrays, axis=axis) + else: + pads = [(0, 0) if d not in pad_widths else pad_widths[d] + for d in self.dims] + array = np.pad(self.data.astype(dtype, copy=False), pads, + mode='constant', constant_values=fill_value) + return type(self)(self.dims, array) + def _roll_one_dim(self, dim, count): axis = self.get_axis_num(dim) @@ -1452,6 +1499,57 @@ def rank(self, dim, pct=False): ranked /= count return Variable(self.dims, ranked) + def rolling_window(self, dim, window, window_dim, center=False, + fill_value=dtypes.NA): + """ + Make a rolling_window along dim and add a new_dim to the last place. + + Parameters + ---------- + dim: str + Dimension over which to compute rolling_window + window: int + Window size of the rolling + window_dim: str + New name of the window dimension. + center: boolean. default False. + If True, pad fill_value for both ends. Otherwise, pad in the head + of the axis. + fill_value: + value to be filled. + + Returns + ------- + Variable that is a view of the original array with a added dimension of + size w. + The return dim: self.dims + (window_dim, ) + The return shape: self.shape + (window, ) + + Examples + -------- + >>> v=Variable(('a', 'b'), np.arange(8).reshape((2,4))) + >>> v.rolling_window(x, 'b', 3, 'window_dim') + + array([[[nan, nan, 0], [nan, 0, 1], [0, 1, 2], [1, 2, 3]], + [[nan, nan, 4], [nan, 4, 5], [4, 5, 6], [5, 6, 7]]]) + + >>> v.rolling_window(x, 'b', 3, 'window_dim', center=True) + + array([[[nan, 0, 1], [0, 1, 2], [1, 2, 3], [2, 3, nan]], + [[nan, 4, 5], [4, 5, 6], [5, 6, 7], [6, 7, nan]]]) + """ + if fill_value is dtypes.NA: # np.nan is passed + dtype, fill_value = dtypes.maybe_promote(self.dtype) + array = self.astype(dtype, copy=False).data + else: + dtype = self.dtype + array = self.data + + new_dims = self.dims + (window_dim, ) + return Variable(new_dims, duck_array_ops.rolling_window( + array, axis=self.get_axis_num(dim), window=window, + center=center, fill_value=fill_value)) + @property def real(self): return type(self)(self.dims, self.data.real, self._attrs) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 095ad5b793b..18fc27c96ab 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2729,7 +2729,7 @@ def test_series_categorical_index(self): if not hasattr(pd, 'CategoricalIndex'): raise unittest.SkipTest('requires pandas with CategoricalIndex') - s = pd.Series(range(5), index=pd.CategoricalIndex(list('aabbc'))) + s = pd.Series(np.arange(5), index=pd.CategoricalIndex(list('aabbc'))) arr = DataArray(s) assert "'a'" in repr(arr) # should not error @@ -3325,9 +3325,11 @@ def da_dask(seed=123): return da +@pytest.mark.parametrize('da', (1, 2), indirect=True) def test_rolling_iter(da): rolling_obj = da.rolling(time=7) + rolling_obj_mean = rolling_obj.mean() assert len(rolling_obj.window_labels) == len(da['time']) assert_identical(rolling_obj.window_labels, da['time']) @@ -3335,6 +3337,16 @@ def test_rolling_iter(da): for i, (label, window_da) in enumerate(rolling_obj): assert label == da['time'].isel(time=i) + actual = rolling_obj_mean.isel(time=i) + expected = window_da.mean('time') + + # TODO add assert_allclose_with_nan, which compares nan position + # as well as the closeness of the values. + assert_array_equal(actual.isnull(), expected.isnull()) + if (~actual.isnull()).sum() > 0: + np.allclose(actual.values[actual.values.nonzero()], + expected.values[expected.values.nonzero()]) + def test_rolling_doc(da): rolling_obj = da.rolling(time=7) @@ -3403,8 +3415,8 @@ def test_rolling_wrapped_bottleneck_dask(da_dask, name, center, min_periods): @pytest.mark.parametrize('center', (True, False)) @pytest.mark.parametrize('min_periods', (None, 1, 2, 3)) @pytest.mark.parametrize('window', (1, 2, 3, 4)) -def test_rolling_pandas_compat(da, center, window, min_periods): - s = pd.Series(range(10)) +def test_rolling_pandas_compat(center, window, min_periods): + s = pd.Series(np.arange(10)) da = DataArray.from_series(s) if min_periods is not None and window < min_periods: @@ -3414,12 +3426,39 @@ def test_rolling_pandas_compat(da, center, window, min_periods): min_periods=min_periods).mean() da_rolling = da.rolling(index=window, center=center, min_periods=min_periods).mean() - # pandas does some fancy stuff in the last position, - # we're not going to do that yet! - np.testing.assert_allclose(s_rolling.values[:-1], - da_rolling.values[:-1]) - np.testing.assert_allclose(s_rolling.index, - da_rolling['index']) + da_rolling_np = da.rolling(index=window, center=center, + min_periods=min_periods).reduce(np.nanmean) + + np.testing.assert_allclose(s_rolling.values, da_rolling.values) + np.testing.assert_allclose(s_rolling.index, da_rolling['index']) + np.testing.assert_allclose(s_rolling.values, da_rolling_np.values) + np.testing.assert_allclose(s_rolling.index, da_rolling_np['index']) + + +@pytest.mark.parametrize('center', (True, False)) +@pytest.mark.parametrize('window', (1, 2, 3, 4)) +def test_rolling_construct(center, window): + s = pd.Series(np.arange(10)) + da = DataArray.from_series(s) + + s_rolling = s.rolling(window, center=center, min_periods=1).mean() + da_rolling = da.rolling(index=window, center=center, min_periods=1) + + da_rolling_mean = da_rolling.construct('window').mean('window') + np.testing.assert_allclose(s_rolling.values, da_rolling_mean.values) + np.testing.assert_allclose(s_rolling.index, da_rolling_mean['index']) + + # with stride + da_rolling_mean = da_rolling.construct('window', + stride=2).mean('window') + np.testing.assert_allclose(s_rolling.values[::2], da_rolling_mean.values) + np.testing.assert_allclose(s_rolling.index[::2], da_rolling_mean['index']) + + # with fill_value + da_rolling_mean = da_rolling.construct( + 'window', stride=2, fill_value=0.0).mean('window') + assert da_rolling_mean.isnull().sum() == 0 + assert (da_rolling_mean == 0.0).sum() >= 0 @pytest.mark.parametrize('da', (1, 2), indirect=True) @@ -3432,6 +3471,10 @@ def test_rolling_reduce(da, center, min_periods, window, name): if min_periods is not None and window < min_periods: min_periods = window + if da.isnull().sum() > 1 and window == 1: + # this causes all nan slices + window = 2 + rolling_obj = da.rolling(time=window, center=center, min_periods=min_periods) @@ -3442,26 +3485,52 @@ def test_rolling_reduce(da, center, min_periods, window, name): assert actual.dims == expected.dims +@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13'), + reason='Old numpy does not support nansum / nanmax for ' + 'object typed arrays.') +@pytest.mark.parametrize('center', (True, False)) +@pytest.mark.parametrize('min_periods', (None, 1, 2, 3)) +@pytest.mark.parametrize('window', (1, 2, 3, 4)) +@pytest.mark.parametrize('name', ('sum', 'max')) +def test_rolling_reduce_nonnumeric(center, min_periods, window, name): + da = DataArray([0, np.nan, 1, 2, np.nan, 3, 4, 5, np.nan, 6, 7], + dims='time').isnull() + + if min_periods is not None and window < min_periods: + min_periods = window + + rolling_obj = da.rolling(time=window, center=center, + min_periods=min_periods) + + # add nan prefix to numpy methods to get similar behavior as bottleneck + actual = rolling_obj.reduce(getattr(np, 'nan%s' % name)) + expected = getattr(rolling_obj, name)() + assert_allclose(actual, expected) + assert actual.dims == expected.dims + + def test_rolling_count_correct(): da = DataArray( [0, np.nan, 1, 2, np.nan, 3, 4, 5, np.nan, 6, 7], dims='time') - result = da.rolling(time=11, min_periods=1).count() - expected = DataArray( - [1, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8], dims='time') - assert_equal(result, expected) - - result = da.rolling(time=11, min_periods=None).count() - expected = DataArray( + kwargs = [{'time': 11, 'min_periods': 1}, + {'time': 11, 'min_periods': None}, + {'time': 7, 'min_periods': 2}] + expecteds = [DataArray( + [1, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8], dims='time'), + DataArray( [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, - np.nan, np.nan, np.nan, np.nan, 8], dims='time') - assert_equal(result, expected) + np.nan, np.nan, np.nan, np.nan, np.nan], dims='time'), + DataArray( + [np.nan, np.nan, 2, 3, 3, 4, 5, 5, 5, 5, 5], dims='time')] + + for kwarg, expected in zip(kwargs, expecteds): + result = da.rolling(**kwarg).count() + assert_equal(result, expected) - result = da.rolling(time=7, min_periods=2).count() - expected = DataArray( - [np.nan, np.nan, 2, 3, 3, 4, 5, 5, 5, 5, 5], dims='time') - assert_equal(result, expected) + result = da.to_dataset(name='var1').rolling(**kwarg).count()['var1'] + assert_equal(result, expected) def test_raise_no_warning_for_nan_in_binary_ops(): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 353128acd39..1b557479ec1 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -4134,12 +4134,36 @@ def test_rolling_pandas_compat(center, window, min_periods): min_periods=min_periods).mean() ds_rolling = ds.rolling(index=window, center=center, min_periods=min_periods).mean() - # pandas does some fancy stuff in the last position, - # we're not going to do that yet! - np.testing.assert_allclose(df_rolling['x'].values[:-1], - ds_rolling['x'].values[:-1]) - np.testing.assert_allclose(df_rolling.index, - ds_rolling['index']) + + np.testing.assert_allclose(df_rolling['x'].values, ds_rolling['x'].values) + np.testing.assert_allclose(df_rolling.index, ds_rolling['index']) + + +@pytest.mark.parametrize('center', (True, False)) +@pytest.mark.parametrize('window', (1, 2, 3, 4)) +def test_rolling_construct(center, window): + df = pd.DataFrame({'x': np.random.randn(20), 'y': np.random.randn(20), + 'time': np.linspace(0, 1, 20)}) + + ds = Dataset.from_dataframe(df) + df_rolling = df.rolling(window, center=center, min_periods=1).mean() + ds_rolling = ds.rolling(index=window, center=center) + + ds_rolling_mean = ds_rolling.construct('window').mean('window') + np.testing.assert_allclose(df_rolling['x'].values, + ds_rolling_mean['x'].values) + np.testing.assert_allclose(df_rolling.index, ds_rolling_mean['index']) + + # with stride + ds_rolling_mean = ds_rolling.construct('window', stride=2).mean('window') + np.testing.assert_allclose(df_rolling['x'][::2].values, + ds_rolling_mean['x'].values) + np.testing.assert_allclose(df_rolling.index[::2], ds_rolling_mean['index']) + # with fill_value + ds_rolling_mean = ds_rolling.construct( + 'window', stride=2, fill_value=0.0).mean('window') + assert ds_rolling_mean.isnull().sum() == 0 + assert (ds_rolling_mean['x'] == 0.0).sum() >= 0 @pytest.mark.slow diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index fde2a1cc726..99250796d8c 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -8,7 +8,8 @@ from xarray import DataArray, concat from xarray.core.duck_array_ops import ( - array_notnull_equiv, concatenate, count, first, last, mean, stack, where) + array_notnull_equiv, concatenate, count, first, last, mean, rolling_window, + stack, where) from xarray.core.pycompat import dask_array_type from xarray.testing import assert_allclose, assert_equal @@ -303,3 +304,28 @@ def test_isnull_with_dask(): da = construct_dataarray(2, np.float32, contains_nan=True, dask=True) assert isinstance(da.isnull().data, dask_array_type) assert_equal(da.isnull().load(), da.load().isnull()) + + +@pytest.mark.skipif(not has_dask, reason='This is for dask.') +@pytest.mark.parametrize('axis', [0, -1]) +@pytest.mark.parametrize('window', [3, 8, 11]) +@pytest.mark.parametrize('center', [True, False]) +def test_dask_rolling(axis, window, center): + import dask.array as da + + x = np.array(np.random.randn(100, 40), dtype=float) + dx = da.from_array(x, chunks=[(6, 30, 30, 20, 14), 8]) + + expected = rolling_window(x, axis=axis, window=window, center=center, + fill_value=np.nan) + actual = rolling_window(dx, axis=axis, window=window, center=center, + fill_value=np.nan) + assert isinstance(actual, da.Array) + assert_array_equal(actual, expected) + assert actual.shape == expected.shape + + # we need to take care of window size if chunk size is small + # window/2 should be smaller than the smallest chunk size. + with pytest.raises(ValueError): + rolling_window(dx, axis=axis, window=100, center=center, + fill_value=np.nan) diff --git a/xarray/tests/test_nputils.py b/xarray/tests/test_nputils.py index 3c9c92ae2ba..d3ef02a039c 100644 --- a/xarray/tests/test_nputils.py +++ b/xarray/tests/test_nputils.py @@ -1,7 +1,8 @@ import numpy as np from numpy.testing import assert_array_equal -from xarray.core.nputils import NumpyVIndexAdapter, _is_contiguous +from xarray.core.nputils import (NumpyVIndexAdapter, _is_contiguous, + rolling_window) def test_is_contiguous(): @@ -28,3 +29,27 @@ def test_vindex(): vindex[[0, 1], [0, 1], :] = vindex[[0, 1], [0, 1], :] vindex[[0, 1], :, [0, 1]] = vindex[[0, 1], :, [0, 1]] vindex[:, [0, 1], [0, 1]] = vindex[:, [0, 1], [0, 1]] + + +def test_rolling(): + x = np.array([1, 2, 3, 4], dtype=float) + + actual = rolling_window(x, axis=-1, window=3, center=True, + fill_value=np.nan) + expected = np.array([[np.nan, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, np.nan]], dtype=float) + assert_array_equal(actual, expected) + + actual = rolling_window(x, axis=-1, window=3, center=False, fill_value=0.0) + expected = np.array([[0, 0, 1], + [0, 1, 2], + [1, 2, 3], + [2, 3, 4]], dtype=float) + assert_array_equal(actual, expected) + + x = np.stack([x, x * 1.1]) + actual = rolling_window(x, axis=-1, window=3, center=False, fill_value=0.0) + expected = np.stack([expected, expected * 1.1], axis=0) + assert_array_equal(actual, expected) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 5f60fc95e15..1373358c476 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -737,6 +737,52 @@ def test_getitem_error(self): with raises_regex(IndexError, 'Dimensions of indexers mis'): v[:, ind] + def test_pad(self): + data = np.arange(4 * 3 * 2).reshape(4, 3, 2) + v = self.cls(['x', 'y', 'z'], data) + + xr_args = [{'x': (2, 1)}, {'y': (0, 3)}, {'x': (3, 1), 'z': (2, 0)}] + np_args = [((2, 1), (0, 0), (0, 0)), ((0, 0), (0, 3), (0, 0)), + ((3, 1), (0, 0), (2, 0))] + for xr_arg, np_arg in zip(xr_args, np_args): + actual = v.pad_with_fill_value(**xr_arg) + expected = np.pad(np.array(v.data.astype(float)), np_arg, + mode='constant', constant_values=np.nan) + assert_array_equal(actual, expected) + assert isinstance(actual._data, type(v._data)) + + # for the boolean array, we pad False + data = np.full_like(data, False, dtype=bool).reshape(4, 3, 2) + v = self.cls(['x', 'y', 'z'], data) + for xr_arg, np_arg in zip(xr_args, np_args): + actual = v.pad_with_fill_value(fill_value=False, **xr_arg) + expected = np.pad(np.array(v.data), np_arg, + mode='constant', constant_values=False) + assert_array_equal(actual, expected) + + def test_rolling_window(self): + # Just a working test. See test_nputils for the algorithm validation + v = self.cls(['x', 'y', 'z'], + np.arange(40 * 30 * 2).reshape(40, 30, 2)) + for (d, w) in [('x', 3), ('y', 5)]: + v_rolling = v.rolling_window(d, w, d + '_window') + assert v_rolling.dims == ('x', 'y', 'z', d + '_window') + assert v_rolling.shape == v.shape + (w, ) + + v_rolling = v.rolling_window(d, w, d + '_window', center=True) + assert v_rolling.dims == ('x', 'y', 'z', d + '_window') + assert v_rolling.shape == v.shape + (w, ) + + # dask and numpy result should be the same + v_loaded = v.load().rolling_window(d, w, d + '_window', + center=True) + assert_array_equal(v_rolling, v_loaded) + + # numpy backend should not be over-written + if isinstance(v._data, np.ndarray): + with pytest.raises(ValueError): + v_loaded[0] = 1.0 + class TestVariable(TestCase, VariableSubclassTestCases): cls = staticmethod(Variable) @@ -1462,7 +1508,7 @@ def test_reduce_funcs(self): v = Variable('t', pd.date_range('2000-01-01', periods=3)) with pytest.raises(NotImplementedError): - v.max(skipna=True) + v.argmax(skipna=True) assert_identical( v.max(), Variable([], pd.Timestamp('2000-01-03'))) @@ -1741,6 +1787,14 @@ def test_getitem_fancy(self): def test_getitem_uint(self): super(TestIndexVariable, self).test_getitem_fancy() + @pytest.mark.xfail + def test_pad(self): + super(TestIndexVariable, self).test_rolling_window() + + @pytest.mark.xfail + def test_rolling_window(self): + super(TestIndexVariable, self).test_rolling_window() + class TestAsCompatibleData(TestCase): def test_unchanged_types(self): From 350e97793f89ddd4097b97e0c4af735a5144be24 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sat, 3 Mar 2018 05:17:28 +0900 Subject: [PATCH 042/282] Fix doc for missing values. (#1950) * Fix doc for missing values. * Remove some methods from api-hidden * Fix for dask page. --- doc/api-hidden.rst | 14 ------------ doc/api.rst | 54 +++++++++++++++++++++++++++------------------ doc/computation.rst | 4 ++-- doc/dask.rst | 2 +- doc/environment.yml | 2 ++ 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/doc/api-hidden.rst b/doc/api-hidden.rst index b8fbfbc288f..f17e3df9e9a 100644 --- a/doc/api-hidden.rst +++ b/doc/api-hidden.rst @@ -22,13 +22,6 @@ Dataset.std Dataset.var - Dataset.isnull - Dataset.notnull - Dataset.count - Dataset.dropna - Dataset.fillna - Dataset.where - core.groupby.DatasetGroupBy.assign core.groupby.DatasetGroupBy.assign_coords core.groupby.DatasetGroupBy.first @@ -68,13 +61,6 @@ DataArray.std DataArray.var - DataArray.isnull - DataArray.notnull - DataArray.count - DataArray.dropna - DataArray.fillna - DataArray.where - core.groupby.DataArrayGroupBy.assign_coords core.groupby.DataArrayGroupBy.first core.groupby.DataArrayGroupBy.last diff --git a/doc/api.rst b/doc/api.rst index 4a26298b268..ae4803e5e62 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -113,6 +113,22 @@ Indexing Dataset.reset_index Dataset.reorder_levels +Missing value handling +---------------------- + +.. autosummary:: + :toctree: generated/ + + Dataset.isnull + Dataset.notnull + Dataset.count + Dataset.dropna + Dataset.fillna + Dataset.ffill + Dataset.bfill + Dataset.interpolate_na + Dataset.where + Computation ----------- @@ -142,17 +158,6 @@ Computation :py:attr:`~Dataset.std` :py:attr:`~Dataset.var` -**Missing values**: -:py:attr:`~Dataset.isnull` -:py:attr:`~Dataset.notnull` -:py:attr:`~Dataset.count` -:py:attr:`~Dataset.dropna` -:py:attr:`~Dataset.fillna` -:py:attr:`~Dataset.ffill` -:py:attr:`~Dataset.bfill` -:py:attr:`~Dataset.interpolate_na` -:py:attr:`~Dataset.where` - **ndarray methods**: :py:attr:`~Dataset.argsort` :py:attr:`~Dataset.clip` @@ -256,6 +261,22 @@ Indexing DataArray.reset_index DataArray.reorder_levels +Missing value handling +---------------------- + +.. autosummary:: + :toctree: generated/ + + DataArray.isnull + DataArray.notnull + DataArray.count + DataArray.dropna + DataArray.fillna + DataArray.ffill + DataArray.bfill + DataArray.interpolate_na + DataArray.where + Comparisons ----------- @@ -296,17 +317,6 @@ Computation :py:attr:`~DataArray.std` :py:attr:`~DataArray.var` -**Missing values**: -:py:attr:`~DataArray.isnull` -:py:attr:`~DataArray.notnull` -:py:attr:`~DataArray.count` -:py:attr:`~DataArray.dropna` -:py:attr:`~DataArray.fillna` -:py:attr:`~DataArray.ffill` -:py:attr:`~DataArray.bfill` -:py:attr:`~DataArray.interpolate_na` -:py:attr:`~DataArray.where` - **ndarray methods**: :py:attr:`~DataArray.argsort` :py:attr:`~DataArray.clip` diff --git a/doc/computation.rst b/doc/computation.rst index 78c645ff8c3..bd0343b214d 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -71,8 +71,8 @@ methods for working with missing data from pandas: x.count() x.dropna(dim='x') x.fillna(-1) - x.ffill() - x.bfill() + x.ffill('x') + x.bfill('x') Like pandas, xarray uses the float value ``np.nan`` (not-a-number) to represent missing values. diff --git a/doc/dask.rst b/doc/dask.rst index 65ebd643e1e..824f30aba4f 100644 --- a/doc/dask.rst +++ b/doc/dask.rst @@ -49,7 +49,7 @@ argument to :py:func:`~xarray.open_dataset` or using the :py:func:`~xarray.open_mfdataset` function. .. ipython:: python - :suppress: + :suppress: import numpy as np import pandas as pd diff --git a/doc/environment.yml b/doc/environment.yml index 5be81e5a314..4ec476e5385 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -6,6 +6,8 @@ dependencies: - python=3.5 - numpy=1.11.2 - pandas=0.21.0 + - scipy=1.0 + - bottleneck - numpydoc=0.7.0 - matplotlib=2.0.0 - seaborn=0.8 From a32475bb21b0efd83c30f36e41346c00f77f4b52 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Sat, 3 Mar 2018 17:41:09 +0100 Subject: [PATCH 043/282] Update some packages on RTD (#1958) * Update nmpy on RTD * update mpl too --- doc/environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/environment.yml b/doc/environment.yml index 4ec476e5385..880151ab2d9 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -3,13 +3,13 @@ channels: - conda-forge - defaults dependencies: - - python=3.5 - - numpy=1.11.2 + - python=3.6 + - numpy=1.13 - pandas=0.21.0 - scipy=1.0 - bottleneck - numpydoc=0.7.0 - - matplotlib=2.0.0 + - matplotlib=2.1.2 - seaborn=0.8 - dask=0.16.0 - ipython=6.2.1 From 4983f1f26070162d274de03971a7b13bb6048490 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 5 Mar 2018 14:14:45 -0800 Subject: [PATCH 044/282] Add x,y kwargs for plot.line(). (#1926) * Add x,y kwargs for plot.line(). Supports both 1D and 2D DataArrays as input. Change variable names to make code clearer: 1. set xplt, yplt to be values that are passed to ax.plot() 2. xlabel, ylabel are axes labels 3. xdim, ydim are dimension names * Respond to comments. * Only allow one of x or y to be specified. * fix docs * Follow suggestions. * Follow suggestions. --- doc/plotting.rst | 10 ++++++ doc/whats-new.rst | 2 ++ xarray/plot/plot.py | 72 ++++++++++++++++++++++++++++----------- xarray/tests/test_plot.py | 39 ++++++++++++++++++++- 4 files changed, 103 insertions(+), 20 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index 2b816a24563..c85a54d783b 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -197,6 +197,16 @@ It is required to explicitly specify either Thus, we could have made the previous plot by specifying ``hue='lat'`` instead of ``x='time'``. If required, the automatic legend can be turned off using ``add_legend=False``. +Dimension along y-axis +~~~~~~~~~~~~~~~~~~~~~~ + +It is also possible to make line plots such that the data are on the x-axis and a dimension is on the y-axis. This can be done by specifying the appropriate ``y`` keyword argument. + +.. ipython:: python + + @savefig plotting_example_xy_kwarg.png + air.isel(time=10, lon=[10, 11]).plot.line(y='lat', hue='lon') + Two Dimensions -------------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index ab667ceba3f..6e819a5d34a 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -46,6 +46,8 @@ Enhancements rolling, windowed rolling, ND-rolling, short-time FFT and convolution. (:issue:`1831`, :issue:`1142`, :issue:`819`) By `Keisuke Fujii `_. +- :py:func:`~plot.line()` learned to make plots with data on x-axis if so specified. (:issue:`575`) + By `Deepak Cherian `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index b5e6d94a4d2..94ddc8c0535 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -176,9 +176,12 @@ def line(darray, *args, **kwargs): Axis on which to plot this figure. By default, use the current axis. Mutually exclusive with ``size`` and ``figsize``. hue : string, optional - Coordinate for which you want multiple lines plotted (2D inputs only). - x : string, optional - Coordinate for x axis. + Coordinate for which you want multiple lines plotted + (2D DataArrays only). + x, y : string, optional + Coordinates for x, y axis. Only one of these may be specified. + The other coordinate plots values from the DataArray on which this + plot method is called. add_legend : boolean, optional Add legend with y axis coordinates (2D inputs only). *args, **kwargs : optional @@ -199,35 +202,66 @@ def line(darray, *args, **kwargs): ax = kwargs.pop('ax', None) hue = kwargs.pop('hue', None) x = kwargs.pop('x', None) + y = kwargs.pop('y', None) add_legend = kwargs.pop('add_legend', True) ax = get_axis(figsize, size, aspect, ax) + error_msg = ('must be either None or one of ({0:s})' + .format(', '.join([repr(dd) for dd in darray.dims]))) + + if x is not None and x not in darray.dims: + raise ValueError('x ' + error_msg) + + if y is not None and y not in darray.dims: + raise ValueError('y ' + error_msg) + + if x is not None and y is not None: + raise ValueError('You cannot specify both x and y kwargs' + 'for line plots.') + if ndims == 1: - xlabel, = darray.dims - if x is not None and xlabel != x: - raise ValueError('Input does not have specified dimension' - ' {!r}'.format(x)) + dim, = darray.dims # get the only dimension name + + if (x is None and y is None) or x == dim: + xplt = darray.coords[dim] + yplt = darray + xlabel = dim + ylabel = darray.name - x = darray.coords[xlabel] + else: + yplt = darray.coords[dim] + xplt = darray + xlabel = darray.name + ylabel = dim else: - if x is None and hue is None: + if x is None and y is None and hue is None: raise ValueError('For 2D inputs, please specify either hue or x.') - xlabel, huelabel = _infer_xy_labels(darray=darray, x=x, y=hue) - x = darray.coords[xlabel] - darray = darray.transpose(xlabel, huelabel) + if y is None: + xlabel, huelabel = _infer_xy_labels(darray=darray, x=x, y=hue) + ylabel = darray.name + xplt = darray.coords[xlabel] + yplt = darray.transpose(xlabel, huelabel) - _ensure_plottable(x) + else: + ylabel, huelabel = _infer_xy_labels(darray=darray, x=y, y=hue) + xlabel = darray.name + xplt = darray.transpose(ylabel, huelabel) + yplt = darray.coords[ylabel] - primitive = ax.plot(x, darray, *args, **kwargs) + _ensure_plottable(xplt) - ax.set_xlabel(xlabel) - ax.set_title(darray._title_for_slice()) + primitive = ax.plot(xplt, yplt, *args, **kwargs) - if darray.name is not None: - ax.set_ylabel(darray.name) + if xlabel is not None: + ax.set_xlabel(xlabel) + + if ylabel is not None: + ax.set_ylabel(ylabel) + + ax.set_title(darray._title_for_slice()) if darray.ndim == 2 and add_legend: ax.legend(handles=primitive, @@ -235,7 +269,7 @@ def line(darray, *args, **kwargs): title=huelabel) # Rotate dates on xlabels - if np.issubdtype(x.dtype, np.datetime64): + if np.issubdtype(xplt.dtype, np.datetime64): ax.get_figure().autofmt_xdate() return primitive diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 26ebcccc748..e509894f06e 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -91,14 +91,42 @@ def setUp(self): def test1d(self): self.darray[:, 0, 0].plot() - with raises_regex(ValueError, 'dimension'): + with raises_regex(ValueError, 'None'): self.darray[:, 0, 0].plot(x='dim_1') + def test_1d_x_y_kw(self): + z = np.arange(10) + da = DataArray(np.cos(z), dims=['z'], coords=[z], name='f') + + xy = [[None, None], + [None, 'z'], + ['z', None]] + + f, ax = plt.subplots(3, 1) + for aa, (x, y) in enumerate(xy): + da.plot(x=x, y=y, ax=ax.flat[aa]) + + with raises_regex(ValueError, 'cannot'): + da.plot(x='z', y='z') + + with raises_regex(ValueError, 'None'): + da.plot(x='f', y='z') + + with raises_regex(ValueError, 'None'): + da.plot(x='z', y='f') + def test_2d_line(self): with raises_regex(ValueError, 'hue'): self.darray[:, :, 0].plot.line() self.darray[:, :, 0].plot.line(hue='dim_1') + self.darray[:, :, 0].plot.line(x='dim_1') + self.darray[:, :, 0].plot.line(y='dim_1') + self.darray[:, :, 0].plot.line(x='dim_0', hue='dim_1') + self.darray[:, :, 0].plot.line(y='dim_0', hue='dim_1') + + with raises_regex(ValueError, 'cannot'): + self.darray[:, :, 0].plot.line(x='dim_1', y='dim_0', hue='dim_1') def test_2d_line_accepts_legend_kw(self): self.darray[:, :, 0].plot.line(x='dim_0', add_legend=False) @@ -279,6 +307,10 @@ def test_xlabel_is_index_name(self): self.darray.plot() assert 'period' == plt.gca().get_xlabel() + def test_no_label_name_on_x_axis(self): + self.darray.plot(y='period') + self.assertEqual('', plt.gca().get_xlabel()) + def test_no_label_name_on_y_axis(self): self.darray.plot() assert '' == plt.gca().get_ylabel() @@ -288,6 +320,11 @@ def test_ylabel_is_data_name(self): self.darray.plot() assert self.darray.name == plt.gca().get_ylabel() + def test_xlabel_is_data_name(self): + self.darray.name = 'temperature' + self.darray.plot(y='period') + self.assertEqual(self.darray.name, plt.gca().get_xlabel()) + def test_format_string(self): self.darray.plot.line('ro') From 3419e9e61beb551850ddc283bb963f09967cb6c3 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 5 Mar 2018 21:05:58 -0800 Subject: [PATCH 045/282] facetgrid: unset cmap if colors is specified. (#1928) --- doc/whats-new.rst | 2 ++ xarray/plot/facetgrid.py | 11 +++++++++++ xarray/tests/test_plot.py | 4 ++++ 3 files changed, 17 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 6e819a5d34a..a1a292ce2a8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -54,6 +54,8 @@ Bug fixes - Fix the precision drop after indexing datetime64 arrays (:issue:`1932`). By `Keisuke Fujii `_. +- Fix kwarg `colors` clashing with auto-inferred `cmap` (:issue:`1461`) + By `Deepak Cherian `_. .. _whats-new.0.10.1: diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index de715094834..5abae214c9f 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -218,6 +218,14 @@ def map_dataarray(self, func, x, y, **kwargs): self : FacetGrid object """ + + cmapkw = kwargs.get('cmap') + colorskw = kwargs.get('colors') + + # colors is mutually exclusive with cmap + if cmapkw and colorskw: + raise ValueError("Can't specify both cmap and colors.") + # These should be consistent with xarray.plot._plot2d cmap_kwargs = {'plot_data': self.data.values, # MPL default @@ -230,6 +238,9 @@ def map_dataarray(self, func, x, y, **kwargs): cmap_params = _determine_cmap_params(**cmap_kwargs) + if colorskw is not None: + cmap_params['cmap'] = None + # Order is important func_kwargs = kwargs.copy() func_kwargs.update(cmap_params) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index e509894f06e..d3409b6aea9 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -938,6 +938,10 @@ def test_facetgrid_cmap(self): # check that all colormaps are the same assert len(set(m.get_cmap().name for m in fg._mappables)) == 1 + def test_cmap_and_color_both(self): + with pytest.raises(ValueError): + self.plotmethod(colors='k', cmap='RdBu') + @pytest.mark.slow class TestContourf(Common2dMixin, PlotTestCase): From 55128aac84cf59906dac1cb3ea4bd643879a5463 Mon Sep 17 00:00:00 2001 From: Kai Pak Date: Tue, 6 Mar 2018 14:00:27 -0500 Subject: [PATCH 046/282] Check for minimum Zarr version. (#1960) * Check for minimum Zarr version. ws * Exclude version check from coverage. additional ws. --- xarray/backends/zarr.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index b0323b51f17..769334992ae 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -2,6 +2,7 @@ from base64 import b64encode from itertools import product +from distutils.version import LooseVersion import numpy as np @@ -271,6 +272,14 @@ class ZarrStore(AbstractWritableDataStore): def open_group(cls, store, mode='r', synchronizer=None, group=None, writer=None): import zarr + min_zarr = '2.2' + + if LooseVersion(zarr.__version__) < min_zarr: # pragma: no cover + raise NotImplementedError("Zarr version %s or greater is " + "required by xarray. See zarr " + "installation " + "http://zarr.readthedocs.io/en/stable/" + "#installation" % min_zarr) zarr_group = zarr.open_group(store=store, mode=mode, synchronizer=synchronizer, path=group) return cls(zarr_group, writer=writer) From 54468e1924174a03e7ead3be8545f687f084f4dd Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Wed, 7 Mar 2018 07:00:56 +0900 Subject: [PATCH 047/282] Vectorized lazy indexing (#1899) * Start working * First support of lazy vectorized indexing. * Some optimization. * Use unique to decompose vectorized indexing. * Consolidate vectorizedIndexing * Support vectorized_indexing in h5py * Refactoring backend array. Added indexing.decompose_indexers. Drop unwrap_explicit_indexers. * typo * bugfix and typo * Fix based on @WeatherGod comments. * Use enum-like object to indicate indexing-support types. * Update test_decompose_indexers. * Bugfix and benchmarks. * fix: support outer/basic indexer in LazilyVectorizedIndexedArray * More comments. * Fixing style errors. * Remove unintended dupicate * combine indexers for on-memory np.ndarray. * fix whats new * fix pydap * Update comments. * Support VectorizedIndexing for rasterio. Some bugfix. * flake8 * More tests * Use LazilyIndexedArray for scalar array instead of loading. * Support negative step slice in rasterio. * Make slice-step always positive * Bugfix in slice-slice * Add pydap support. * Rename LazilyIndexedArray -> LazilyOuterIndexedArray. Remove duplicate in zarr.py * flake8 * Added transpose to LazilyOuterIndexedArray --- asv_bench/benchmarks/dataset_io.py | 47 +++- doc/whats-new.rst | 5 + xarray/backends/h5netcdf_.py | 19 +- xarray/backends/netCDF4_.py | 15 +- xarray/backends/pydap_.py | 14 +- xarray/backends/pynio_.py | 15 +- xarray/backends/rasterio_.py | 67 +++-- xarray/backends/zarr.py | 30 +-- xarray/conventions.py | 2 +- xarray/core/indexing.py | 407 ++++++++++++++++++++++++++--- xarray/core/variable.py | 5 +- xarray/tests/test_backends.py | 236 ++++++++++------- xarray/tests/test_dataset.py | 3 +- xarray/tests/test_indexing.py | 226 +++++++++++++--- xarray/tests/test_variable.py | 37 +-- 15 files changed, 859 insertions(+), 269 deletions(-) diff --git a/asv_bench/benchmarks/dataset_io.py b/asv_bench/benchmarks/dataset_io.py index de6c34b5af3..54ed9ac9fa2 100644 --- a/asv_bench/benchmarks/dataset_io.py +++ b/asv_bench/benchmarks/dataset_io.py @@ -5,7 +5,7 @@ import xarray as xr -from . import randn, requires_dask +from . import randn, randint, requires_dask try: import dask @@ -71,6 +71,15 @@ def make_ds(self): self.ds.attrs = {'history': 'created for xarray benchmarking'} + self.oinds = {'time': randint(0, self.nt, 120), + 'lon': randint(0, self.nx, 20), + 'lat': randint(0, self.ny, 10)} + self.vinds = {'time': xr.DataArray(randint(0, self.nt, 120), + dims='x'), + 'lon': xr.DataArray(randint(0, self.nx, 120), + dims='x'), + 'lat': slice(3, 20)} + class IOWriteSingleNetCDF3(IOSingleNetCDF): def setup(self): @@ -98,6 +107,14 @@ def setup(self): def time_load_dataset_netcdf4(self): xr.open_dataset(self.filepath, engine='netcdf4').load() + def time_orthogonal_indexing(self): + ds = xr.open_dataset(self.filepath, engine='netcdf4') + ds = ds.isel(**self.oinds).load() + + def time_vectorized_indexing(self): + ds = xr.open_dataset(self.filepath, engine='netcdf4') + ds = ds.isel(**self.vinds).load() + class IOReadSingleNetCDF3(IOReadSingleNetCDF4): def setup(self): @@ -111,6 +128,14 @@ def setup(self): def time_load_dataset_scipy(self): xr.open_dataset(self.filepath, engine='scipy').load() + def time_orthogonal_indexing(self): + ds = xr.open_dataset(self.filepath, engine='scipy') + ds = ds.isel(**self.oinds).load() + + def time_vectorized_indexing(self): + ds = xr.open_dataset(self.filepath, engine='scipy') + ds = ds.isel(**self.vinds).load() + class IOReadSingleNetCDF4Dask(IOSingleNetCDF): def setup(self): @@ -127,6 +152,16 @@ def time_load_dataset_netcdf4_with_block_chunks(self): xr.open_dataset(self.filepath, engine='netcdf4', chunks=self.block_chunks).load() + def time_load_dataset_netcdf4_with_block_chunks_oindexing(self): + ds = xr.open_dataset(self.filepath, engine='netcdf4', + chunks=self.block_chunks) + ds = ds.isel(**self.oinds).load() + + def time_load_dataset_netcdf4_with_block_chunks_vindexing(self): + ds = xr.open_dataset(self.filepath, engine='netcdf4', + chunks=self.block_chunks) + ds = ds.isel(**self.vinds).load() + def time_load_dataset_netcdf4_with_block_chunks_multiprocessing(self): with dask.set_options(get=dask.multiprocessing.get): xr.open_dataset(self.filepath, engine='netcdf4', @@ -158,6 +193,16 @@ def time_load_dataset_scipy_with_block_chunks(self): xr.open_dataset(self.filepath, engine='scipy', chunks=self.block_chunks).load() + def time_load_dataset_scipy_with_block_chunks_oindexing(self): + ds = xr.open_dataset(self.filepath, engine='scipy', + chunks=self.block_chunks) + ds = ds.isel(**self.oinds).load() + + def time_load_dataset_scipy_with_block_chunks_vindexing(self): + ds = xr.open_dataset(self.filepath, engine='scipy', + chunks=self.block_chunks) + ds = ds.isel(**self.vinds).load() + def time_load_dataset_scipy_with_time_chunks(self): with dask.set_options(get=dask.multiprocessing.get): xr.open_dataset(self.filepath, engine='scipy', diff --git a/doc/whats-new.rst b/doc/whats-new.rst index a1a292ce2a8..1621f7edcb7 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -38,6 +38,11 @@ Documentation Enhancements ~~~~~~~~~~~~ +- Support lazy vectorized-indexing. After this change, flexible indexing such + as orthogonal/vectorized indexing, becomes possible for all the backend + arrays. Also, lazy ``transpose`` is now also supported. (:issue:`1897`) + By `Keisuke Fujii `_. + - Improve :py:func:`~xarray.DataArray.rolling` logic. :py:func:`~xarray.DataArrayRolling` object now supports :py:func:`~xarray.DataArrayRolling.construct` method that returns a view diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 4e70c8858c3..1d166f05eb1 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -16,15 +16,20 @@ class H5NetCDFArrayWrapper(BaseNetCDF4Array): def __getitem__(self, key): - key = indexing.unwrap_explicit_indexer( - key, self, allow=(indexing.BasicIndexer, indexing.OuterIndexer)) + key, np_inds = indexing.decompose_indexer( + key, self.shape, indexing.IndexingSupport.OUTER_1VECTOR) + # h5py requires using lists for fancy indexing: # https://github.com/h5py/h5py/issues/992 - # OuterIndexer only holds 1D integer ndarrays, so it's safe to convert - # them to lists. - key = tuple(list(k) if isinstance(k, np.ndarray) else k for k in key) + key = tuple(list(k) if isinstance(k, np.ndarray) else k for k in + key.tuple) with self.datastore.ensure_open(autoclose=True): - return self.get_array()[key] + array = self.get_array()[key] + + if len(np_inds.tuple) > 0: + array = indexing.NumpyIndexingAdapter(array)[np_inds] + + return array def maybe_decode_bytes(txt): @@ -85,7 +90,7 @@ def __init__(self, filename, mode='r', format=None, group=None, def open_store_variable(self, name, var): with self.ensure_open(autoclose=False): dimensions = var.dimensions - data = indexing.LazilyIndexedArray( + data = indexing.LazilyOuterIndexedArray( H5NetCDFArrayWrapper(name, self)) attrs = _read_attributes(var) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 313539bd6bc..4903e9a98f2 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -48,9 +48,8 @@ def get_array(self): class NetCDF4ArrayWrapper(BaseNetCDF4Array): def __getitem__(self, key): - key = indexing.unwrap_explicit_indexer( - key, self, allow=(indexing.BasicIndexer, indexing.OuterIndexer)) - + key, np_inds = indexing.decompose_indexer( + key, self.shape, indexing.IndexingSupport.OUTER) if self.datastore.is_remote: # pragma: no cover getitem = functools.partial(robust_getitem, catch=RuntimeError) else: @@ -58,7 +57,7 @@ def __getitem__(self, key): with self.datastore.ensure_open(autoclose=True): try: - data = getitem(self.get_array(), key) + array = getitem(self.get_array(), key.tuple) except IndexError: # Catch IndexError in netCDF4 and return a more informative # error message. This is most often called when an unsorted @@ -71,7 +70,10 @@ def __getitem__(self, key): msg += '\n\nOriginal traceback:\n' + traceback.format_exc() raise IndexError(msg) - return data + if len(np_inds.tuple) > 0: + array = indexing.NumpyIndexingAdapter(array)[np_inds] + + return array def _encode_nc4_variable(var): @@ -277,7 +279,8 @@ def open(cls, filename, mode='r', format='NETCDF4', group=None, def open_store_variable(self, name, var): with self.ensure_open(autoclose=False): dimensions = var.dimensions - data = indexing.LazilyIndexedArray(NetCDF4ArrayWrapper(name, self)) + data = indexing.LazilyOuterIndexedArray( + NetCDF4ArrayWrapper(name, self)) attributes = OrderedDict((k, var.getncattr(k)) for k in var.ncattrs()) _ensure_fill_value_valid(data, attributes) diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index a16b1ddcbc8..4a932e3dad2 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -22,18 +22,22 @@ def dtype(self): return self.array.dtype def __getitem__(self, key): - key = indexing.unwrap_explicit_indexer( - key, target=self, allow=indexing.BasicIndexer) + key, np_inds = indexing.decompose_indexer( + key, self.shape, indexing.IndexingSupport.BASIC) # pull the data from the array attribute if possible, to avoid # downloading coordinate data twice array = getattr(self.array, 'array', self.array) - result = robust_getitem(array, key, catch=ValueError) + result = robust_getitem(array, key.tuple, catch=ValueError) # pydap doesn't squeeze axes automatically like numpy - axis = tuple(n for n, k in enumerate(key) + axis = tuple(n for n, k in enumerate(key.tuple) if isinstance(k, integer_types)) if len(axis) > 0: result = np.squeeze(result, axis) + + if len(np_inds.tuple) > 0: + result = indexing.NumpyIndexingAdapter(np.asarray(result))[np_inds] + return result @@ -74,7 +78,7 @@ def open(cls, url, session=None): return cls(ds) def open_store_variable(self, var): - data = indexing.LazilyIndexedArray(PydapArrayWrapper(var)) + data = indexing.LazilyOuterIndexedArray(PydapArrayWrapper(var)) return Variable(var.dimensions, data, _fix_attributes(var.attributes)) diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 30969fcd9a0..95226e453b4 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -24,14 +24,19 @@ def get_array(self): return self.datastore.ds.variables[self.variable_name] def __getitem__(self, key): - key = indexing.unwrap_explicit_indexer( - key, target=self, allow=indexing.BasicIndexer) + key, np_inds = indexing.decompose_indexer( + key, self.shape, indexing.IndexingSupport.BASIC) with self.datastore.ensure_open(autoclose=True): array = self.get_array() - if key == () and self.ndim == 0: + if key.tuple == () and self.ndim == 0: return array.get_value() - return array[key] + + array = array[key.tuple] + if len(np_inds.tuple) > 0: + array = indexing.NumpyIndexingAdapter(array)[np_inds] + + return array class NioDataStore(AbstractDataStore, DataStorePickleMixin): @@ -51,7 +56,7 @@ def __init__(self, filename, mode='r', autoclose=False): self._mode = mode def open_store_variable(self, name, var): - data = indexing.LazilyIndexedArray(NioArrayWrapper(name, self)) + data = indexing.LazilyOuterIndexedArray(NioArrayWrapper(name, self)) return Variable(var.dimensions, data, var.attributes) def get_variables(self): diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 8777f6e7053..f856f9f9e13 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -42,48 +42,73 @@ def dtype(self): def shape(self): return self._shape - def __getitem__(self, key): - key = indexing.unwrap_explicit_indexer( - key, self, allow=(indexing.BasicIndexer, indexing.OuterIndexer)) + def _get_indexer(self, key): + """ Get indexer for rasterio array. + + Parameter + --------- + key: ExplicitIndexer + + Returns + ------- + band_key: an indexer for the 1st dimension + window: two tuples. Each consists of (start, stop). + squeeze_axis: axes to be squeezed + np_ind: indexer for loaded numpy array + + See also + -------- + indexing.decompose_indexer + """ + key, np_inds = indexing.decompose_indexer( + key, self.shape, indexing.IndexingSupport.OUTER) # bands cannot be windowed but they can be listed - band_key = key[0] - n_bands = self.shape[0] + band_key = key.tuple[0] + new_shape = [] + np_inds2 = [] + # bands (axis=0) cannot be windowed but they can be listed if isinstance(band_key, slice): - start, stop, step = band_key.indices(n_bands) - if step is not None and step != 1: - raise IndexError(_ERROR_MSG) - band_key = np.arange(start, stop) + start, stop, step = band_key.indices(self.shape[0]) + band_key = np.arange(start, stop, step) # be sure we give out a list band_key = (np.asarray(band_key) + 1).tolist() + if isinstance(band_key, list): # if band_key is not a scalar + new_shape.append(len(band_key)) + np_inds2.append(slice(None)) # but other dims can only be windowed window = [] squeeze_axis = [] - for i, (k, n) in enumerate(zip(key[1:], self.shape[1:])): + for i, (k, n) in enumerate(zip(key.tuple[1:], self.shape[1:])): if isinstance(k, slice): + # step is always positive. see indexing.decompose_indexer start, stop, step = k.indices(n) - if step is not None and step != 1: - raise IndexError(_ERROR_MSG) + np_inds2.append(slice(None, None, step)) + new_shape.append(stop - start) elif is_scalar(k): # windowed operations will always return an array # we will have to squeeze it later - squeeze_axis.append(i + 1) + squeeze_axis.append(- (2 - i)) start = k stop = k + 1 else: - k = np.asarray(k) - start = k[0] - stop = k[-1] + 1 - ids = np.arange(start, stop) - if not ((k.shape == ids.shape) and np.all(k == ids)): - raise IndexError(_ERROR_MSG) + start, stop = np.min(k), np.max(k) + 1 + np_inds2.append(k - start) + new_shape.append(stop - start) window.append((start, stop)) + np_inds = indexing._combine_indexers( + indexing.OuterIndexer(tuple(np_inds2)), new_shape, np_inds) + return band_key, window, tuple(squeeze_axis), np_inds + + def __getitem__(self, key): + band_key, window, squeeze_axis, np_inds = self._get_indexer(key) + out = self.rasterio_ds.read(band_key, window=tuple(window)) if squeeze_axis: out = np.squeeze(out, axis=squeeze_axis) - return out + return indexing.NumpyIndexingAdapter(out)[np_inds] def _parse_envi(meta): @@ -249,7 +274,7 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, else: attrs[k] = v - data = indexing.LazilyIndexedArray(RasterioArrayWrapper(riods)) + data = indexing.LazilyOuterIndexedArray(RasterioArrayWrapper(riods)) # this lets you write arrays loaded with rasterio data = indexing.CopyOnWriteArray(data) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 769334992ae..8797e3104a1 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -42,30 +42,6 @@ def _ensure_valid_fill_value(value, dtype): return _encode_zarr_attr_value(valid) -def _replace_slices_with_arrays(key, shape): - """Replace slice objects in vindex with equivalent ndarray objects.""" - num_slices = sum(1 for k in key if isinstance(k, slice)) - ndims = [k.ndim for k in key if isinstance(k, np.ndarray)] - array_subspace_size = max(ndims) if ndims else 0 - assert len(key) == len(shape) - new_key = [] - slice_count = 0 - for k, size in zip(key, shape): - if isinstance(k, slice): - # the slice subspace always appears after the ndarray subspace - array = np.arange(*k.indices(size)) - sl = [np.newaxis] * len(shape) - sl[array_subspace_size + slice_count] = slice(None) - k = array[tuple(sl)] - slice_count += 1 - else: - assert isinstance(k, np.ndarray) - k = k[(slice(None),) * array_subspace_size + - (np.newaxis,) * num_slices] - new_key.append(k) - return tuple(new_key) - - class ZarrArrayWrapper(BackendArray): def __init__(self, variable_name, datastore): self.datastore = datastore @@ -85,8 +61,8 @@ def __getitem__(self, key): if isinstance(key, indexing.BasicIndexer): return array[key.tuple] elif isinstance(key, indexing.VectorizedIndexer): - return array.vindex[_replace_slices_with_arrays(key.tuple, - self.shape)] + return array.vindex[indexing._arrayize_vectorized_indexer( + key.tuple, self.shape).tuple] else: assert isinstance(key, indexing.OuterIndexer) return array.oindex[key.tuple] @@ -301,7 +277,7 @@ def __init__(self, zarr_group, writer=None): super(ZarrStore, self).__init__(zarr_writer) def open_store_variable(self, name, zarr_array): - data = indexing.LazilyIndexedArray(ZarrArrayWrapper(name, self)) + data = indexing.LazilyOuterIndexedArray(ZarrArrayWrapper(name, self)) dimensions, attributes = _get_zarr_dims_and_attrs(zarr_array, _DIMENSION_KEY) attributes = OrderedDict(attributes) diff --git a/xarray/conventions.py b/xarray/conventions.py index 5bcbd83ee90..93fd56ed5d2 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -490,7 +490,7 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True, del attributes['dtype'] data = BoolTypeArray(data) - return Variable(dimensions, indexing.LazilyIndexedArray(data), + return Variable(dimensions, indexing.LazilyOuterIndexedArray(data), attributes, encoding=encoding) diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 0d55eed894e..bd16618d766 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -254,7 +254,7 @@ def slice_slice(old_slice, applied_slice, size): items = _expand_slice(old_slice, size)[applied_slice] if len(items) > 0: start = items[0] - stop = items[-1] + step + stop = items[-1] + int(np.sign(step)) if stop < 0: stop = None else: @@ -425,23 +425,6 @@ def __array__(self, dtype=None): return np.asarray(self[key], dtype=dtype) -def unwrap_explicit_indexer(key, target, allow): - """Unwrap an explicit key into a tuple.""" - if not isinstance(key, ExplicitIndexer): - raise TypeError('unexpected key type: {}'.format(key)) - if not isinstance(key, allow): - key_type_name = { - BasicIndexer: 'Basic', - OuterIndexer: 'Outer', - VectorizedIndexer: 'Vectorized' - }[type(key)] - raise NotImplementedError( - '{} indexing for {} is not implemented. Load your data first with ' - '.load(), .compute() or .persist(), or disable caching by setting ' - 'cache=False in open_dataset.'.format(key_type_name, type(target))) - return key.tuple - - class ImplicitToExplicitIndexingAdapter(utils.NDArrayMixin): """Wrap an array, converting tuples into the indicated explicit indexer.""" @@ -457,8 +440,8 @@ def __getitem__(self, key): return self.array[self.indexer_cls(key)] -class LazilyIndexedArray(ExplicitlyIndexedNDArrayMixin): - """Wrap an array to make basic and orthogonal indexing lazy. +class LazilyOuterIndexedArray(ExplicitlyIndexedNDArrayMixin): + """Wrap an array to make basic and outer indexing lazy. """ def __init__(self, array, key=None): @@ -483,13 +466,6 @@ def __init__(self, array, key=None): self.key = key def _updated_key(self, new_key): - # TODO should suport VectorizedIndexer - if isinstance(new_key, VectorizedIndexer): - raise NotImplementedError( - 'Vectorized indexing for {} is not implemented. Load your ' - 'data first with .load() or .compute(), or disable caching by ' - 'setting cache=False in open_dataset.'.format(type(self))) - iter_new_key = iter(expanded_indexer(new_key.tuple, self.ndim)) full_key = [] for size, k in zip(self.array.shape, self.key.tuple): @@ -517,10 +493,21 @@ def __array__(self, dtype=None): array = as_indexable(self.array) return np.asarray(array[self.key], dtype=None) + def transpose(self, order): + return LazilyVectorizedIndexedArray( + self.array, self.key).transpose(order) + def __getitem__(self, indexer): + if isinstance(indexer, VectorizedIndexer): + array = LazilyVectorizedIndexedArray(self.array, self.key) + return array[indexer] return type(self)(self.array, self._updated_key(indexer)) def __setitem__(self, key, value): + if isinstance(key, VectorizedIndexer): + raise NotImplementedError( + 'Lazy item assignment with the vectorized indexer is not yet ' + 'implemented. Load your data first by .load() or compute().') full_key = self._updated_key(key) self.array[full_key] = value @@ -529,6 +516,56 @@ def __repr__(self): (type(self).__name__, self.array, self.key)) +class LazilyVectorizedIndexedArray(ExplicitlyIndexedNDArrayMixin): + """Wrap an array to make vectorized indexing lazy. + """ + + def __init__(self, array, key): + """ + Parameters + ---------- + array : array_like + Array like object to index. + key : VectorizedIndexer + """ + if isinstance(key, (BasicIndexer, OuterIndexer)): + self.key = _outer_to_vectorized_indexer(key, array.shape) + else: + self.key = _arrayize_vectorized_indexer(key, array.shape) + self.array = as_indexable(array) + + @property + def shape(self): + return np.broadcast(*self.key.tuple).shape + + def __array__(self, dtype=None): + return np.asarray(self.array[self.key], dtype=None) + + def _updated_key(self, new_key): + return _combine_indexers(self.key, self.shape, new_key) + + def __getitem__(self, indexer): + # If the indexed array becomes a scalar, return LazilyOuterIndexedArray + if all(isinstance(ind, integer_types) for ind in indexer.tuple): + key = BasicIndexer(tuple(k[indexer.tuple] for k in self.key.tuple)) + return LazilyOuterIndexedArray(self.array, key) + return type(self)(self.array, self._updated_key(indexer)) + + def transpose(self, order): + key = VectorizedIndexer(tuple( + k.transpose(order) for k in self.key.tuple)) + return type(self)(self.array, key) + + def __setitem__(self, key, value): + raise NotImplementedError( + 'Lazy item assignment with the vectorized indexer is not yet ' + 'implemented. Load your data first by .load() or compute().') + + def __repr__(self): + return ('%s(array=%r, key=%r)' % + (type(self).__name__, self.array, self.key)) + + def _wrap_numpy_scalars(array): """Wrap NumPy scalars in 0d arrays.""" if np.isscalar(array): @@ -553,6 +590,9 @@ def __array__(self, dtype=None): def __getitem__(self, key): return type(self)(_wrap_numpy_scalars(self.array[key])) + def transpose(self, order): + return self.array.transpose(order) + def __setitem__(self, key, value): self._ensure_copied() self.array[key] = value @@ -573,6 +613,9 @@ def __array__(self, dtype=None): def __getitem__(self, key): return type(self)(_wrap_numpy_scalars(self.array[key])) + def transpose(self, order): + return self.array.transpose(order) + def __setitem__(self, key, value): self.array[key] = value @@ -599,24 +642,26 @@ def _outer_to_vectorized_indexer(key, shape): Parameters ---------- - key : tuple - Tuple from an OuterIndexer to convert. + key : Outer/Basic Indexer + An indexer to convert. shape : tuple Shape of the array subject to the indexing. Returns ------- - tuple + VectorizedIndexer Tuple suitable for use to index a NumPy array with vectorized indexing. - Each element is an integer or array: broadcasting them together gives - the shape of the result. + Each element is an array: broadcasting them together gives the shape + of the result. """ + key = key.tuple + n_dim = len([k for k in key if not isinstance(k, integer_types)]) i_dim = 0 new_key = [] for k, size in zip(key, shape): if isinstance(k, integer_types): - new_key.append(k) + new_key.append(np.array(k).reshape((1,) * n_dim)) else: # np.ndarray or slice if isinstance(k, slice): k = np.arange(*k.indices(size)) @@ -625,7 +670,7 @@ def _outer_to_vectorized_indexer(key, shape): (1,) * (n_dim - i_dim - 1)] new_key.append(k.reshape(*shape)) i_dim += 1 - return tuple(new_key) + return VectorizedIndexer(tuple(new_key)) def _outer_to_numpy_indexer(key, shape): @@ -633,8 +678,8 @@ def _outer_to_numpy_indexer(key, shape): Parameters ---------- - key : tuple - Tuple from an OuterIndexer to convert. + key : Basic/OuterIndexer + An indexer to convert. shape : tuple Shape of the array subject to the indexing. @@ -643,13 +688,284 @@ def _outer_to_numpy_indexer(key, shape): tuple Tuple suitable for use to index a NumPy array. """ - if len([k for k in key if not isinstance(k, slice)]) <= 1: + if len([k for k in key.tuple if not isinstance(k, slice)]) <= 1: # If there is only one vector and all others are slice, # it can be safely used in mixed basic/advanced indexing. # Boolean index should already be converted to integer array. - return tuple(key) + return key.tuple else: - return _outer_to_vectorized_indexer(key, shape) + return _outer_to_vectorized_indexer(key, shape).tuple + + +def _combine_indexers(old_key, shape, new_key): + """ Combine two indexers. + + Parameters + ---------- + old_key: ExplicitIndexer + The first indexer for the original array + shape: tuple of ints + Shape of the original array to be indexed by old_key + new_key: + The second indexer for indexing original[old_key] + """ + if not isinstance(old_key, VectorizedIndexer): + old_key = _outer_to_vectorized_indexer(old_key, shape) + if len(old_key.tuple) == 0: + return new_key + + new_shape = np.broadcast(*old_key.tuple).shape + if isinstance(new_key, VectorizedIndexer): + new_key = _arrayize_vectorized_indexer(new_key, new_shape) + else: + new_key = _outer_to_vectorized_indexer(new_key, new_shape) + + return VectorizedIndexer(tuple(o[new_key.tuple] for o in + np.broadcast_arrays(*old_key.tuple))) + + +class IndexingSupport(object): # could inherit from enum.Enum on Python 3 + # for backends that support only basic indexer + BASIC = 'BASIC' + # for backends that support basic / outer indexer + OUTER = 'OUTER' + # for backends that support outer indexer including at most 1 vector. + OUTER_1VECTOR = 'OUTER_1VECTOR' + # for backends that support full vectorized indexer. + VECTORIZED = 'VECTORIZED' + + +def decompose_indexer(indexer, shape, indexing_support): + if isinstance(indexer, VectorizedIndexer): + return _decompose_vectorized_indexer(indexer, shape, indexing_support) + if isinstance(indexer, (BasicIndexer, OuterIndexer)): + return _decompose_outer_indexer(indexer, shape, indexing_support) + raise TypeError('unexpected key type: {}'.format(indexer)) + + +def _decompose_slice(key, size): + """ convert a slice to successive two slices. The first slice always has + a positive step. + """ + start, stop, step = key.indices(size) + if step > 0: + # If key already has a positive step, use it as is in the backend + return key, slice(None) + else: + # determine stop precisely for step > 1 case + # e.g. [98:2:-2] -> [98:3:-2] + stop = start + int((stop - start - 1) / step) * step + 1 + start, stop = stop + 1, start + 1 + return slice(start, stop, -step), slice(None, None, -1) + + +def _decompose_vectorized_indexer(indexer, shape, indexing_support): + """ + Decompose vectorized indexer to the successive two indexers, where the + first indexer will be used to index backend arrays, while the second one + is used to index loaded on-memory np.ndarray. + + Parameters + ---------- + indexer: VectorizedIndexer + indexing_support: one of IndexerSupport entries + + Returns + ------- + backend_indexer: OuterIndexer or BasicIndexer + np_indexers: an ExplicitIndexer (VectorizedIndexer / BasicIndexer) + + Note + ---- + This function is used to realize the vectorized indexing for the backend + arrays that only support basic or outer indexing. + + As an example, let us consider to index a few elements from a backend array + with a vectorized indexer ([0, 3, 1], [2, 3, 2]). + Even if the backend array only supports outer indexing, it is more + efficient to load a subslice of the array than loading the entire array, + + >>> backend_indexer = OuterIndexer([0, 1, 3], [2, 3]) + >>> array = array[backend_indexer] # load subslice of the array + >>> np_indexer = VectorizedIndexer([0, 2, 1], [0, 1, 0]) + >>> array[np_indexer] # vectorized indexing for on-memory np.ndarray. + """ + assert isinstance(indexer, VectorizedIndexer) + + if indexing_support is IndexingSupport.VECTORIZED: + return indexer, BasicIndexer(()) + + backend_indexer = [] + np_indexer = [] + # convert negative indices + indexer = [np.where(k < 0, k + s, k) if isinstance(k, np.ndarray) else k + for k, s in zip(indexer.tuple, shape)] + + for k, s in zip(indexer, shape): + if isinstance(k, slice): + # If it is a slice, then we will slice it as-is + # (but make its step positive) in the backend, + # and then use all of it (slice(None)) for the in-memory portion. + bk_slice, np_slice = _decompose_slice(k, s) + backend_indexer.append(bk_slice) + np_indexer.append(np_slice) + else: + # If it is a (multidimensional) np.ndarray, just pickup the used + # keys without duplication and store them as a 1d-np.ndarray. + oind, vind = np.unique(k, return_inverse=True) + backend_indexer.append(oind) + np_indexer.append(vind.reshape(*k.shape)) + + backend_indexer = OuterIndexer(tuple(backend_indexer)) + np_indexer = VectorizedIndexer(tuple(np_indexer)) + + if indexing_support is IndexingSupport.OUTER: + return backend_indexer, np_indexer + + # If the backend does not support outer indexing, + # backend_indexer (OuterIndexer) is also decomposed. + backend_indexer, np_indexer1 = _decompose_outer_indexer( + backend_indexer, shape, indexing_support) + np_indexer = _combine_indexers(np_indexer1, shape, np_indexer) + return backend_indexer, np_indexer + + +def _decompose_outer_indexer(indexer, shape, indexing_support): + """ + Decompose outer indexer to the successive two indexers, where the + first indexer will be used to index backend arrays, while the second one + is used to index the loaded on-memory np.ndarray. + + Parameters + ---------- + indexer: VectorizedIndexer + indexing_support: One of the entries of IndexingSupport + + Returns + ------- + backend_indexer: OuterIndexer or BasicIndexer + np_indexers: an ExplicitIndexer (OuterIndexer / BasicIndexer) + + Note + ---- + This function is used to realize the vectorized indexing for the backend + arrays that only support basic or outer indexing. + + As an example, let us consider to index a few elements from a backend array + with a orthogonal indexer ([0, 3, 1], [2, 3, 2]). + Even if the backend array only supports basic indexing, it is more + efficient to load a subslice of the array than loading the entire array, + + >>> backend_indexer = BasicIndexer(slice(0, 3), slice(2, 3)) + >>> array = array[backend_indexer] # load subslice of the array + >>> np_indexer = OuterIndexer([0, 2, 1], [0, 1, 0]) + >>> array[np_indexer] # outer indexing for on-memory np.ndarray. + """ + if indexing_support == IndexingSupport.VECTORIZED: + return indexer, BasicIndexer(()) + assert isinstance(indexer, (OuterIndexer, BasicIndexer)) + + backend_indexer = [] + np_indexer = [] + # make indexer positive + pos_indexer = [] + for k, s in zip(indexer.tuple, shape): + if isinstance(k, np.ndarray): + pos_indexer.append(np.where(k < 0, k + s, k)) + elif isinstance(k, integer_types) and k < 0: + pos_indexer.append(k + s) + else: + pos_indexer.append(k) + indexer = pos_indexer + + if indexing_support is IndexingSupport.OUTER_1VECTOR: + # some backends such as h5py supports only 1 vector in indexers + # We choose the most efficient axis + gains = [(np.max(k) - np.min(k) + 1.0) / len(np.unique(k)) + if isinstance(k, np.ndarray) else 0 for k in indexer] + array_index = np.argmax(np.array(gains)) if len(gains) > 0 else None + + for i, (k, s) in enumerate(zip(indexer, shape)): + if isinstance(k, np.ndarray) and i != array_index: + # np.ndarray key is converted to slice that covers the entire + # entries of this key. + backend_indexer.append(slice(np.min(k), np.max(k) + 1)) + np_indexer.append(k - np.min(k)) + elif isinstance(k, np.ndarray): + # Remove duplicates and sort them in the increasing order + pkey, ekey = np.unique(k, return_inverse=True) + backend_indexer.append(pkey) + np_indexer.append(ekey) + elif isinstance(k, integer_types): + backend_indexer.append(k) + else: # slice: convert positive step slice for backend + bk_slice, np_slice = _decompose_slice(k, s) + backend_indexer.append(bk_slice) + np_indexer.append(np_slice) + + return (OuterIndexer(tuple(backend_indexer)), + OuterIndexer(tuple(np_indexer))) + + if indexing_support == IndexingSupport.OUTER: + for k, s in zip(indexer, shape): + if isinstance(k, slice): + # slice: convert positive step slice for backend + bk_slice, np_slice = _decompose_slice(k, s) + backend_indexer.append(bk_slice) + np_indexer.append(np_slice) + elif isinstance(k, integer_types): + backend_indexer.append(k) + elif isinstance(k, np.ndarray) and (np.diff(k) >= 0).all(): + backend_indexer.append(k) + np_indexer.append(slice(None)) + else: + # Remove duplicates and sort them in the increasing order + oind, vind = np.unique(k, return_inverse=True) + backend_indexer.append(oind) + np_indexer.append(vind.reshape(*k.shape)) + + return (OuterIndexer(tuple(backend_indexer)), + OuterIndexer(tuple(np_indexer))) + + # basic indexer + assert indexing_support == IndexingSupport.BASIC + + for k, s in zip(indexer, shape): + if isinstance(k, np.ndarray): + # np.ndarray key is converted to slice that covers the entire + # entries of this key. + backend_indexer.append(slice(np.min(k), np.max(k) + 1)) + np_indexer.append(k - np.min(k)) + elif isinstance(k, integer_types): + backend_indexer.append(k) + else: # slice: convert positive step slice for backend + bk_slice, np_slice = _decompose_slice(k, s) + backend_indexer.append(bk_slice) + np_indexer.append(np_slice) + + return (BasicIndexer(tuple(backend_indexer)), + OuterIndexer(tuple(np_indexer))) + + +def _arrayize_vectorized_indexer(indexer, shape): + """ Return an identical vindex but slices are replaced by arrays """ + slices = [v for v in indexer.tuple if isinstance(v, slice)] + if len(slices) == 0: + return indexer + + arrays = [v for v in indexer.tuple if isinstance(v, np.ndarray)] + n_dim = arrays[0].ndim if len(arrays) > 0 else 0 + i_dim = 0 + new_key = [] + for v, size in zip(indexer.tuple, shape): + if isinstance(v, np.ndarray): + new_key.append(np.reshape(v, v.shape + (1, ) * len(slices))) + else: # slice + shape = ((1,) * (n_dim + i_dim) + (-1,) + + (1,) * (len(slices) - i_dim - 1)) + new_key.append(np.arange(*v.indices(size)).reshape(shape)) + i_dim += 1 + return VectorizedIndexer(tuple(new_key)) def _dask_array_with_chunks_hint(array, chunks): @@ -698,7 +1014,7 @@ def create_mask(indexer, shape, chunks_hint=None): same shape as the indexing result. """ if isinstance(indexer, OuterIndexer): - key = _outer_to_vectorized_indexer(indexer.tuple, shape) + key = _outer_to_vectorized_indexer(indexer, shape).tuple assert not any(isinstance(k, slice) for k in key) mask = _masked_result_drop_slice(key, chunks_hint) @@ -793,7 +1109,7 @@ def _ensure_ndarray(self, value): def _indexing_array_and_key(self, key): if isinstance(key, OuterIndexer): array = self.array - key = _outer_to_numpy_indexer(key.tuple, self.array.shape) + key = _outer_to_numpy_indexer(key, self.array.shape) elif isinstance(key, VectorizedIndexer): array = nputils.NumpyVIndexAdapter(self.array) key = key.tuple @@ -805,6 +1121,9 @@ def _indexing_array_and_key(self, key): return array, key + def transpose(self, order): + return self.array.transpose(order) + def __getitem__(self, key): array, key = self._indexing_array_and_key(key) return self._ensure_ndarray(array[key]) @@ -848,6 +1167,9 @@ def __setitem__(self, key, value): 'into memory explicitly using the .load() ' 'method or accessing its .values attribute.') + def transpose(self, order): + return self.array.transpose(order) + class PandasIndexAdapter(ExplicitlyIndexedNDArrayMixin): """Wrap a pandas.Index to preserve dtypes and handle explicit indexing.""" @@ -922,6 +1244,9 @@ def __getitem__(self, indexer): return result + def transpose(self, order): + return self.array # self.array should be always one-dimensional + def __repr__(self): return ('%s(array=%r, dtype=%r)' % (type(self).__name__, self.array, self.dtype)) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index bb4285fba0a..66a4e781161 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -121,7 +121,7 @@ def _maybe_wrap_data(data): Put pandas.Index and numpy.ndarray arguments in adapter objects to ensure they can be indexed properly. - NumpyArrayAdapter, PandasIndexAdapter and LazilyIndexedArray should + NumpyArrayAdapter, PandasIndexAdapter and LazilyOuterIndexedArray should all pass through unmodified. """ if isinstance(data, pd.Index): @@ -1053,7 +1053,8 @@ def transpose(self, *dims): axes = self.get_axis_num(dims) if len(dims) < 2: # no need to transpose if only one dimension return self.copy(deep=False) - data = duck_array_ops.transpose(self.data, axes) + + data = as_indexable(self._data).transpose(axes) return type(self)(dims, data, self._attrs, self._encoding, fastpath=True) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 32c79107a2c..21152829096 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -403,34 +403,86 @@ def test_orthogonal_indexing(self): 'dim3': np.arange(5)} expected = in_memory.isel(**indexers) actual = on_disk.isel(**indexers) + # make sure the array is not yet loaded into memory + assert not actual['var1'].variable._in_memory assert_identical(expected, actual) # do it twice, to make sure we're switched from orthogonal -> numpy # when we cached the values actual = on_disk.isel(**indexers) assert_identical(expected, actual) - def _test_vectorized_indexing(self, vindex_support=True): - # Make sure vectorized_indexing works or at least raises - # NotImplementedError + def test_vectorized_indexing(self): in_memory = create_test_data() with self.roundtrip(in_memory) as on_disk: indexers = {'dim1': DataArray([0, 2, 0], dims='a'), 'dim2': DataArray([0, 2, 3], dims='a')} expected = in_memory.isel(**indexers) - if vindex_support: - actual = on_disk.isel(**indexers) - assert_identical(expected, actual) - # do it twice, to make sure we're switched from - # orthogonal -> numpy when we cached the values - actual = on_disk.isel(**indexers) - assert_identical(expected, actual) - else: - with raises_regex(NotImplementedError, 'Vectorized indexing '): - actual = on_disk.isel(**indexers) + actual = on_disk.isel(**indexers) + # make sure the array is not yet loaded into memory + assert not actual['var1'].variable._in_memory + assert_identical(expected, actual.load()) + # do it twice, to make sure we're switched from + # vectorized -> numpy when we cached the values + actual = on_disk.isel(**indexers) + assert_identical(expected, actual) - def test_vectorized_indexing(self): - # This test should be overwritten if vindex is supported - self._test_vectorized_indexing(vindex_support=False) + def multiple_indexing(indexers): + # make sure a sequence of lazy indexings certainly works. + with self.roundtrip(in_memory) as on_disk: + actual = on_disk['var3'] + expected = in_memory['var3'] + for ind in indexers: + actual = actual.isel(**ind) + expected = expected.isel(**ind) + # make sure the array is not yet loaded into memory + assert not actual.variable._in_memory + assert_identical(expected, actual.load()) + + # two-staged vectorized-indexing + indexers = [ + {'dim1': DataArray([[0, 7], [2, 6], [3, 5]], dims=['a', 'b']), + 'dim3': DataArray([[0, 4], [1, 3], [2, 2]], dims=['a', 'b'])}, + {'a': DataArray([0, 1], dims=['c']), + 'b': DataArray([0, 1], dims=['c'])} + ] + multiple_indexing(indexers) + + # vectorized-slice mixed + indexers = [ + {'dim1': DataArray([[0, 7], [2, 6], [3, 5]], dims=['a', 'b']), + 'dim3': slice(None, 10)} + ] + multiple_indexing(indexers) + + # vectorized-integer mixed + indexers = [ + {'dim3': 0}, + {'dim1': DataArray([[0, 7], [2, 6], [3, 5]], dims=['a', 'b'])}, + {'a': slice(None, None, 2)} + ] + multiple_indexing(indexers) + + # vectorized-integer mixed + indexers = [ + {'dim3': 0}, + {'dim1': DataArray([[0, 7], [2, 6], [3, 5]], dims=['a', 'b'])}, + {'a': 1, 'b': 0} + ] + multiple_indexing(indexers) + + # with negative step slice. + indexers = [ + {'dim1': DataArray([[0, 7], [2, 6], [3, 5]], dims=['a', 'b']), + 'dim3': slice(-1, 1, -1)}, + ] + multiple_indexing(indexers) + + # with negative step slice. + indexers = [ + {'dim1': DataArray([[0, 7], [2, 6], [3, 5]], dims=['a', 'b']), + 'dim3': slice(-1, 1, -2)}, + ] + multiple_indexing(indexers) def test_isel_dataarray(self): # Make sure isel works lazily. GH:issue:1688 @@ -508,7 +560,6 @@ def test_roundtrip_bytes_with_fill_value(self): encoding = {'_FillValue': b'X', 'dtype': 'S1'} original = Dataset({'x': ('t', values, {}, encoding)}) expected = original.copy(deep=True) - print(original) with self.roundtrip(original) as actual: assert_identical(expected, actual) @@ -756,9 +807,6 @@ def test_append_with_invalid_dim_raises(self): 'Unable to update size for existing dimension'): self.save(data, tmp_file, mode='a') - def test_vectorized_indexing(self): - self._test_vectorized_indexing(vindex_support=False) - def test_multiindex_not_implemented(self): ds = (Dataset(coords={'y': ('x', [1, 2]), 'z': ('x', ['a', 'b'])}) .set_index(x=['y', 'z'])) @@ -1119,9 +1167,6 @@ def test_dataset_caching(self): # caching behavior differs for dask pass - def test_vectorized_indexing(self): - self._test_vectorized_indexing(vindex_support=True) - class NetCDF4ViaDaskDataTestAutocloseTrue(NetCDF4ViaDaskDataTest): autoclose = True @@ -1238,9 +1283,6 @@ def test_chunk_encoding_with_dask(self): with self.roundtrip(ds_chunk4) as actual: pass - def test_vectorized_indexing(self): - self._test_vectorized_indexing(vindex_support=True) - def test_hidden_zarr_keys(self): expected = create_test_data() with self.create_store() as store: @@ -1360,27 +1402,6 @@ def create_zarr_target(self): yield tmp -def test_replace_slices_with_arrays(): - (actual,) = xr.backends.zarr._replace_slices_with_arrays( - key=(slice(None),), shape=(5,)) - np.testing.assert_array_equal(actual, np.arange(5)) - - actual = xr.backends.zarr._replace_slices_with_arrays( - key=(np.arange(5),) * 3, shape=(8, 10, 12)) - expected = np.stack([np.arange(5)] * 3) - np.testing.assert_array_equal(np.stack(actual), expected) - - a, b = xr.backends.zarr._replace_slices_with_arrays( - key=(np.arange(5), slice(None)), shape=(8, 10)) - np.testing.assert_array_equal(a, np.arange(5)[:, np.newaxis]) - np.testing.assert_array_equal(b, np.arange(10)[np.newaxis, :]) - - a, b = xr.backends.zarr._replace_slices_with_arrays( - key=(slice(None), np.arange(5)), shape=(8, 10)) - np.testing.assert_array_equal(a, np.arange(8)[np.newaxis, :]) - np.testing.assert_array_equal(b, np.arange(5)[:, np.newaxis]) - - @requires_scipy class ScipyInMemoryDataTest(CFEncodedDataTest, NetCDF3Only, TestCase): engine = 'scipy' @@ -1589,19 +1610,6 @@ def create_store(self): with create_tmp_file() as tmp_file: yield backends.H5NetCDFStore(tmp_file, 'w') - def test_orthogonal_indexing(self): - # simplified version for h5netcdf - in_memory = create_test_data() - with self.roundtrip(in_memory) as on_disk: - indexers = {'dim3': np.arange(5)} - expected = in_memory.isel(**indexers) - actual = on_disk.isel(**indexers) - assert_identical(expected, actual.load()) - - def test_array_type_after_indexing(self): - # h5netcdf does not support multiple list-like indexers - pass - def test_complex(self): expected = Dataset({'x': ('y', np.ones(5) + 1j * np.ones(5))}) with self.roundtrip(expected) as actual: @@ -2049,9 +2057,6 @@ def test_dataarray_compute(self): self.assertTrue(computed._in_memory) assert_allclose(actual, computed, decode_bytes=False) - def test_vectorized_indexing(self): - self._test_vectorized_indexing(vindex_support=True) - class DaskTestAutocloseTrue(DaskTest): autoclose = True @@ -2111,6 +2116,17 @@ def test_cmp_local_file(self): assert_equal(actual.isel(j=slice(1, 2)), expected.isel(j=slice(1, 2))) + with self.create_datasets() as (actual, expected): + indexers = {'i': [1, 0, 0], 'j': [1, 2, 0, 1]} + assert_equal(actual.isel(**indexers), + expected.isel(**indexers)) + + with self.create_datasets() as (actual, expected): + indexers = {'i': DataArray([0, 1, 0], dims='a'), + 'j': DataArray([0, 2, 1], dims='a')} + assert_equal(actual.isel(**indexers), + expected.isel(**indexers)) + def test_compatible_to_netcdf(self): # make sure it can be saved as a netcdf with self.create_datasets() as (actual, expected): @@ -2155,19 +2171,6 @@ def test_write_store(self): # pynio is read-only for now pass - def test_orthogonal_indexing(self): - # pynio also does not support list-like indexing - with raises_regex(NotImplementedError, 'Outer indexing'): - super(TestPyNio, self).test_orthogonal_indexing() - - def test_isel_dataarray(self): - with raises_regex(NotImplementedError, 'Outer indexing'): - super(TestPyNio, self).test_isel_dataarray() - - def test_array_type_after_indexing(self): - # pynio also does not support list-like indexing - pass - @contextlib.contextmanager def open(self, path, **kwargs): with open_dataset(path, engine='pynio', autoclose=self.autoclose, @@ -2346,17 +2349,62 @@ def test_indexing(self): # tests # assert_allclose checks all data + coordinates assert_allclose(actual, expected) - - # Slicing - ex = expected.isel(x=slice(2, 5), y=slice(5, 7)) - ac = actual.isel(x=slice(2, 5), y=slice(5, 7)) - assert_allclose(ac, ex) - - ex = expected.isel(band=slice(1, 2), x=slice(2, 5), - y=slice(5, 7)) - ac = actual.isel(band=slice(1, 2), x=slice(2, 5), - y=slice(5, 7)) - assert_allclose(ac, ex) + assert not actual.variable._in_memory + + # Basic indexer + ind = {'x': slice(2, 5), 'y': slice(5, 7)} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + ind = {'band': slice(1, 2), 'x': slice(2, 5), 'y': slice(5, 7)} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + ind = {'band': slice(1, 2), 'x': slice(2, 5), 'y': 0} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + # orthogonal indexer + ind = {'band': np.array([2, 1, 0]), + 'x': np.array([1, 0]), 'y': np.array([0, 2])} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + ind = {'band': np.array([2, 1, 0]), + 'x': np.array([1, 0]), 'y': 0} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + # minus-stepped slice + ind = {'band': np.array([2, 1, 0]), + 'x': slice(-1, None, -1), 'y': 0} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + ind = {'band': np.array([2, 1, 0]), + 'x': 1, 'y': slice(-1, 1, -2)} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + # None is selected + ind = {'band': np.array([2, 1, 0]), + 'x': 1, 'y': slice(2, 2, 1)} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + # vectorized indexer + ind = {'band': DataArray([2, 1, 0], dims='a'), + 'x': DataArray([1, 0, 0], dims='a'), + 'y': np.array([0, 2])} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + + ind = { + 'band': DataArray([[2, 1, 0], [1, 0, 2]], dims=['a', 'b']), + 'x': DataArray([[1, 0, 0], [0, 1, 0]], dims=['a', 'b']), + 'y': 0} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory # Selecting lists of bands is fine ex = expected.isel(band=[1, 2]) @@ -2366,15 +2414,6 @@ def test_indexing(self): ac = actual.isel(band=[0, 2]) assert_allclose(ac, ex) - # but on x and y only windowed operations are allowed, more - # exotic slicing should raise an error - err_msg = 'not valid on rasterio' - with raises_regex(IndexError, err_msg): - actual.isel(x=[2, 4], y=[1, 3]).values - with raises_regex(IndexError, err_msg): - actual.isel(x=[4, 2]).values - with raises_regex(IndexError, err_msg): - actual.isel(x=slice(5, 2, -1)).values # Integer indexing ex = expected.isel(band=1) ac = actual.isel(band=1) @@ -2412,11 +2451,6 @@ def test_caching(self): # Cache is the default with xr.open_rasterio(tmp_file) as actual: - # Without cache an error is raised - err_msg = 'not valid on rasterio' - with raises_regex(IndexError, err_msg): - actual.isel(x=[2, 4]).values - # This should cache everything assert_allclose(actual, expected) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 1b557479ec1..e3e2934a4fd 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -77,7 +77,8 @@ def get_variables(self): def lazy_inaccessible(k, v): if k in self._indexvars: return v - data = indexing.LazilyIndexedArray(InaccessibleArray(v.values)) + data = indexing.LazilyOuterIndexedArray( + InaccessibleArray(v.values)) return Variable(v.dims, data, v.attrs) return dict((k, lazy_inaccessible(k, v)) for k, v in iteritems(self._variables)) diff --git a/xarray/tests/test_indexing.py b/xarray/tests/test_indexing.py index 4884eebe759..0d1045d35c0 100644 --- a/xarray/tests/test_indexing.py +++ b/xarray/tests/test_indexing.py @@ -136,22 +136,24 @@ def test_indexer(data, x, expected_pos, expected_idx=None): class TestLazyArray(TestCase): def test_slice_slice(self): I = ReturnItem() # noqa: E741 # allow ambiguous name - x = np.arange(100) - slices = [I[:3], I[:4], I[2:4], I[:1], I[:-1], I[5:-1], I[-5:-1], - I[::-1], I[5::-1], I[:3:-1], I[:30:-1], I[10:4:], I[::4], - I[4:4:4], I[:4:-4]] - for i in slices: - for j in slices: - expected = x[i][j] - new_slice = indexing.slice_slice(i, j, size=100) - actual = x[new_slice] - assert_array_equal(expected, actual) + for size in [100, 99]: + # We test even/odd size cases + x = np.arange(size) + slices = [I[:3], I[:4], I[2:4], I[:1], I[:-1], I[5:-1], I[-5:-1], + I[::-1], I[5::-1], I[:3:-1], I[:30:-1], I[10:4:], I[::4], + I[4:4:4], I[:4:-4], I[::-2]] + for i in slices: + for j in slices: + expected = x[i][j] + new_slice = indexing.slice_slice(i, j, size=size) + actual = x[new_slice] + assert_array_equal(expected, actual) def test_lazily_indexed_array(self): original = np.random.rand(10, 20, 30) x = indexing.NumpyIndexingAdapter(original) v = Variable(['i', 'j', 'k'], original) - lazy = indexing.LazilyIndexedArray(x) + lazy = indexing.LazilyOuterIndexedArray(x) v_lazy = Variable(['i', 'j', 'k'], lazy) I = ReturnItem() # noqa: E741 # allow ambiguous name # test orthogonally applied indexers @@ -170,7 +172,7 @@ def test_lazily_indexed_array(self): assert expected.shape == actual.shape assert_array_equal(expected, actual) assert isinstance(actual._data, - indexing.LazilyIndexedArray) + indexing.LazilyOuterIndexedArray) # make sure actual.key is appropriate type if all(isinstance(k, native_int_types + (slice, )) @@ -185,14 +187,66 @@ def test_lazily_indexed_array(self): indexers = [(3, 2), (I[:], 0), (I[:2], -1), (I[:4], [0]), ([4, 5], 0), ([0, 1, 2], [0, 1]), ([0, 3, 5], I[:2])] for i, j in indexers: - expected = np.asarray(v[i][j]) + expected = v[i][j] actual = v_lazy[i][j] assert expected.shape == actual.shape assert_array_equal(expected, actual) - assert isinstance(actual._data, indexing.LazilyIndexedArray) + + # test transpose + if actual.ndim > 1: + order = np.random.choice(actual.ndim, actual.ndim) + order = np.array(actual.dims) + transposed = actual.transpose(*order) + assert_array_equal(expected.transpose(*order), transposed) + assert isinstance( + actual._data, (indexing.LazilyVectorizedIndexedArray, + indexing.LazilyOuterIndexedArray)) + + assert isinstance(actual._data, indexing.LazilyOuterIndexedArray) assert isinstance(actual._data.array, indexing.NumpyIndexingAdapter) + def test_vectorized_lazily_indexed_array(self): + original = np.random.rand(10, 20, 30) + x = indexing.NumpyIndexingAdapter(original) + v_eager = Variable(['i', 'j', 'k'], x) + lazy = indexing.LazilyOuterIndexedArray(x) + v_lazy = Variable(['i', 'j', 'k'], lazy) + I = ReturnItem() # noqa: E741 # allow ambiguous name + + def check_indexing(v_eager, v_lazy, indexers): + for indexer in indexers: + actual = v_lazy[indexer] + expected = v_eager[indexer] + assert expected.shape == actual.shape + assert isinstance(actual._data, + (indexing.LazilyVectorizedIndexedArray, + indexing.LazilyOuterIndexedArray)) + assert_array_equal(expected, actual) + v_eager = expected + v_lazy = actual + + # test orthogonal indexing + indexers = [(I[:], 0, 1), (Variable('i', [0, 1]), )] + check_indexing(v_eager, v_lazy, indexers) + + # vectorized indexing + indexers = [ + (Variable('i', [0, 1]), Variable('i', [0, 1]), slice(None)), + (slice(1, 3, 2), 0)] + check_indexing(v_eager, v_lazy, indexers) + + indexers = [ + (slice(None, None, 2), 0, slice(None, 10)), + (Variable('i', [3, 2, 4, 3]), Variable('i', [3, 2, 1, 0])), + (Variable(['i', 'j'], [[0, 1], [1, 2]]), )] + check_indexing(v_eager, v_lazy, indexers) + + indexers = [ + (Variable('i', [3, 2, 4, 3]), Variable('i', [3, 2, 1, 0])), + (Variable(['i', 'j'], [[0, 1], [1, 2]]), )] + check_indexing(v_eager, v_lazy, indexers) + class TestCopyOnWriteArray(TestCase): def test_setitem(self): @@ -220,19 +274,19 @@ def test_index_scalar(self): class TestMemoryCachedArray(TestCase): def test_wrapper(self): - original = indexing.LazilyIndexedArray(np.arange(10)) + original = indexing.LazilyOuterIndexedArray(np.arange(10)) wrapped = indexing.MemoryCachedArray(original) assert_array_equal(wrapped, np.arange(10)) assert isinstance(wrapped.array, indexing.NumpyIndexingAdapter) def test_sub_array(self): - original = indexing.LazilyIndexedArray(np.arange(10)) + original = indexing.LazilyOuterIndexedArray(np.arange(10)) wrapped = indexing.MemoryCachedArray(original) child = wrapped[B[:5]] assert isinstance(child, indexing.MemoryCachedArray) assert_array_equal(child, np.arange(5)) assert isinstance(child.array, indexing.NumpyIndexingAdapter) - assert isinstance(wrapped.array, indexing.LazilyIndexedArray) + assert isinstance(wrapped.array, indexing.LazilyOuterIndexedArray) def test_setitem(self): original = np.arange(10) @@ -331,21 +385,126 @@ def test_vectorized_indexer(): np.arange(5, dtype=np.int64))) -def test_unwrap_explicit_indexer(): - indexer = indexing.BasicIndexer((1, 2)) - target = None - - unwrapped = indexing.unwrap_explicit_indexer( - indexer, target, allow=indexing.BasicIndexer) - assert unwrapped == (1, 2) - - with raises_regex(NotImplementedError, 'Load your data'): - indexing.unwrap_explicit_indexer( - indexer, target, allow=indexing.OuterIndexer) - - with raises_regex(TypeError, 'unexpected key type'): - indexing.unwrap_explicit_indexer( - indexer.tuple, target, allow=indexing.OuterIndexer) +class Test_vectorized_indexer(TestCase): + def setUp(self): + self.data = indexing.NumpyIndexingAdapter(np.random.randn(10, 12, 13)) + self.indexers = [np.array([[0, 3, 2], ]), + np.array([[0, 3, 3], [4, 6, 7]]), + slice(2, -2, 2), slice(2, -2, 3), slice(None)] + + def test_arrayize_vectorized_indexer(self): + for i, j, k in itertools.product(self.indexers, repeat=3): + vindex = indexing.VectorizedIndexer((i, j, k)) + vindex_array = indexing._arrayize_vectorized_indexer( + vindex, self.data.shape) + np.testing.assert_array_equal( + self.data[vindex], self.data[vindex_array],) + + actual = indexing._arrayize_vectorized_indexer( + indexing.VectorizedIndexer((slice(None),)), shape=(5,)) + np.testing.assert_array_equal(actual.tuple, [np.arange(5)]) + + actual = indexing._arrayize_vectorized_indexer( + indexing.VectorizedIndexer((np.arange(5),) * 3), shape=(8, 10, 12)) + expected = np.stack([np.arange(5)] * 3) + np.testing.assert_array_equal(np.stack(actual.tuple), expected) + + actual = indexing._arrayize_vectorized_indexer( + indexing.VectorizedIndexer((np.arange(5), slice(None))), + shape=(8, 10)) + a, b = actual.tuple + np.testing.assert_array_equal(a, np.arange(5)[:, np.newaxis]) + np.testing.assert_array_equal(b, np.arange(10)[np.newaxis, :]) + + actual = indexing._arrayize_vectorized_indexer( + indexing.VectorizedIndexer((slice(None), np.arange(5))), + shape=(8, 10)) + a, b = actual.tuple + np.testing.assert_array_equal(a, np.arange(8)[np.newaxis, :]) + np.testing.assert_array_equal(b, np.arange(5)[:, np.newaxis]) + + +def get_indexers(shape, mode): + if mode == 'vectorized': + indexed_shape = (3, 4) + indexer = tuple(np.random.randint(0, s, size=indexed_shape) + for s in shape) + return indexing.VectorizedIndexer(indexer) + + elif mode == 'outer': + indexer = tuple(np.random.randint(0, s, s + 2) for s in shape) + return indexing.OuterIndexer(indexer) + + elif mode == 'outer_scalar': + indexer = (np.random.randint(0, 3, 4), 0, slice(None, None, 2)) + return indexing.OuterIndexer(indexer[:len(shape)]) + + elif mode == 'outer_scalar2': + indexer = (np.random.randint(0, 3, 4), -2, slice(None, None, 2)) + return indexing.OuterIndexer(indexer[:len(shape)]) + + elif mode == 'outer1vec': + indexer = [slice(2, -3) for s in shape] + indexer[1] = np.random.randint(0, shape[1], shape[1] + 2) + return indexing.OuterIndexer(tuple(indexer)) + + elif mode == 'basic': # basic indexer + indexer = [slice(2, -3) for s in shape] + indexer[0] = 3 + return indexing.BasicIndexer(tuple(indexer)) + + elif mode == 'basic1': # basic indexer + return indexing.BasicIndexer((3, )) + + elif mode == 'basic2': # basic indexer + indexer = [0, 2, 4] + return indexing.BasicIndexer(tuple(indexer[:len(shape)])) + + elif mode == 'basic3': # basic indexer + indexer = [slice(None) for s in shape] + indexer[0] = slice(-2, 2, -2) + indexer[1] = slice(1, -1, 2) + return indexing.BasicIndexer(tuple(indexer[:len(shape)])) + + +@pytest.mark.parametrize('size', [100, 99]) +@pytest.mark.parametrize('sl', [slice(1, -1, 1), slice(None, -1, 2), + slice(-1, 1, -1), slice(-1, 1, -2)]) +def test_decompose_slice(size, sl): + x = np.arange(size) + slice1, slice2 = indexing._decompose_slice(sl, size) + expected = x[sl] + actual = x[slice1][slice2] + assert_array_equal(expected, actual) + + +@pytest.mark.parametrize('shape', [(10, 5, 8), (10, 3)]) +@pytest.mark.parametrize('indexer_mode', + ['vectorized', 'outer', 'outer_scalar', + 'outer_scalar2', 'outer1vec', + 'basic', 'basic1', 'basic2', 'basic3']) +@pytest.mark.parametrize('indexing_support', + [indexing.IndexingSupport.BASIC, + indexing.IndexingSupport.OUTER, + indexing.IndexingSupport.OUTER_1VECTOR, + indexing.IndexingSupport.VECTORIZED]) +def test_decompose_indexers(shape, indexer_mode, indexing_support): + data = np.random.randn(*shape) + indexer = get_indexers(shape, indexer_mode) + + backend_ind, np_ind = indexing.decompose_indexer( + indexer, shape, indexing_support) + + expected = indexing.NumpyIndexingAdapter(data)[indexer] + array = indexing.NumpyIndexingAdapter(data)[backend_ind] + if len(np_ind.tuple) > 0: + array = indexing.NumpyIndexingAdapter(array)[np_ind] + np.testing.assert_array_equal(expected, array) + + if not all(isinstance(k, indexing.integer_types) for k in np_ind.tuple): + combined_ind = indexing._combine_indexers(backend_ind, shape, np_ind) + array = indexing.NumpyIndexingAdapter(data)[combined_ind] + np.testing.assert_array_equal(expected, array) def test_implicit_indexing_adapter(): @@ -382,7 +541,8 @@ def nonzero(x): expected_data = np.moveaxis(expected_data, old_order, new_order) - outer_index = (nonzero(i), nonzero(j), nonzero(k)) + outer_index = indexing.OuterIndexer((nonzero(i), nonzero(j), + nonzero(k))) actual = indexing._outer_to_numpy_indexer(outer_index, v.shape) actual_data = v.data[actual] np.testing.assert_array_equal(actual_data, expected_data) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 1373358c476..c4489f50246 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -16,9 +16,9 @@ from xarray.core import indexing from xarray.core.common import full_like, ones_like, zeros_like from xarray.core.indexing import ( - BasicIndexer, CopyOnWriteArray, DaskIndexingAdapter, LazilyIndexedArray, - MemoryCachedArray, NumpyIndexingAdapter, OuterIndexer, PandasIndexAdapter, - VectorizedIndexer) + BasicIndexer, CopyOnWriteArray, DaskIndexingAdapter, + LazilyOuterIndexedArray, MemoryCachedArray, NumpyIndexingAdapter, + OuterIndexer, PandasIndexAdapter, VectorizedIndexer) from xarray.core.pycompat import PY3, OrderedDict from xarray.core.utils import NDArrayMixin from xarray.core.variable import as_compatible_data, as_variable @@ -988,9 +988,9 @@ def test_repr(self): assert expected == repr(v) def test_repr_lazy_data(self): - v = Variable('x', LazilyIndexedArray(np.arange(2e5))) + v = Variable('x', LazilyOuterIndexedArray(np.arange(2e5))) assert '200000 values with dtype' in repr(v) - assert isinstance(v._data, LazilyIndexedArray) + assert isinstance(v._data, LazilyOuterIndexedArray) def test_detect_indexer_type(self): """ Tests indexer type was correctly detected. """ @@ -1798,7 +1798,7 @@ def test_rolling_window(self): class TestAsCompatibleData(TestCase): def test_unchanged_types(self): - types = (np.asarray, PandasIndexAdapter, indexing.LazilyIndexedArray) + types = (np.asarray, PandasIndexAdapter, LazilyOuterIndexedArray) for t in types: for data in [np.arange(3), pd.date_range('2000-01-01', periods=3), @@ -1961,18 +1961,19 @@ def test_NumpyIndexingAdapter(self): v = Variable(dims=('x', 'y'), data=NumpyIndexingAdapter( NumpyIndexingAdapter(self.d))) - def test_LazilyIndexedArray(self): - v = Variable(dims=('x', 'y'), data=LazilyIndexedArray(self.d)) + def test_LazilyOuterIndexedArray(self): + v = Variable(dims=('x', 'y'), data=LazilyOuterIndexedArray(self.d)) self.check_orthogonal_indexing(v) - with raises_regex(NotImplementedError, 'Vectorized indexing for '): - self.check_vectorized_indexing(v) + self.check_vectorized_indexing(v) # doubly wrapping - v = Variable(dims=('x', 'y'), - data=LazilyIndexedArray(LazilyIndexedArray(self.d))) + v = Variable( + dims=('x', 'y'), + data=LazilyOuterIndexedArray(LazilyOuterIndexedArray(self.d))) self.check_orthogonal_indexing(v) # hierarchical wrapping - v = Variable(dims=('x', 'y'), - data=LazilyIndexedArray(NumpyIndexingAdapter(self.d))) + v = Variable( + dims=('x', 'y'), + data=LazilyOuterIndexedArray(NumpyIndexingAdapter(self.d))) self.check_orthogonal_indexing(v) def test_CopyOnWriteArray(self): @@ -1980,11 +1981,11 @@ def test_CopyOnWriteArray(self): self.check_orthogonal_indexing(v) self.check_vectorized_indexing(v) # doubly wrapping - v = Variable(dims=('x', 'y'), - data=CopyOnWriteArray(LazilyIndexedArray(self.d))) + v = Variable( + dims=('x', 'y'), + data=CopyOnWriteArray(LazilyOuterIndexedArray(self.d))) self.check_orthogonal_indexing(v) - with raises_regex(NotImplementedError, 'Vectorized indexing for '): - self.check_vectorized_indexing(v) + self.check_vectorized_indexing(v) def test_MemoryCachedArray(self): v = Variable(dims=('x', 'y'), data=MemoryCachedArray(self.d)) From 870e4eaf1895cfeffdc27dab61ad739e67133777 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 7 Mar 2018 08:41:53 -0800 Subject: [PATCH 048/282] Better error message for vectorize=True in apply_ufunc with old numpy (#1963) * Better error message for vectorize=True in apply_ufunc with old numpy * Typo otype -> otypes * add missing __future__ imports * all_output_core_dims -> all_core_dims --- doc/whats-new.rst | 3 +++ xarray/core/computation.py | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 1621f7edcb7..57ae66818d3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -57,6 +57,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- Raise an informative error message when using ``apply_ufunc`` with numpy + v1.11 (:issue:`1956`). + By `Stephan Hoyer `_. - Fix the precision drop after indexing datetime64 arrays (:issue:`1932`). By `Keisuke Fujii `_. - Fix kwarg `colors` clashing with auto-inferred `cmap` (:issue:`1461`) diff --git a/xarray/core/computation.py b/xarray/core/computation.py index b7590ab6b4b..858936aad6c 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -1,8 +1,8 @@ """ Functions for applying functions that act on arrays to xarray's labeled data. - -NOT PUBLIC API. """ +from __future__ import absolute_import, division, print_function +from distutils.version import LooseVersion import functools import itertools import operator @@ -882,10 +882,21 @@ def earth_mover_distance(first_samples, func = functools.partial(func, **kwargs_) if vectorize: - func = np.vectorize(func, - otypes=output_dtypes, - signature=signature.to_gufunc_string(), - excluded=set(kwargs)) + if signature.all_core_dims: + # we need the signature argument + if LooseVersion(np.__version__) < '1.12': # pragma: no cover + raise NotImplementedError( + 'numpy 1.12 or newer required when using vectorize=True ' + 'in xarray.apply_ufunc with non-scalar output core ' + 'dimensions.') + func = np.vectorize(func, + otypes=output_dtypes, + signature=signature.to_gufunc_string(), + excluded=set(kwargs)) + else: + func = np.vectorize(func, + otypes=output_dtypes, + excluded=set(kwargs)) variables_ufunc = functools.partial(apply_variable_ufunc, func, signature=signature, From 9accae01b5b4018e05385269780708bcf8048d16 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Fri, 9 Mar 2018 10:51:45 +1100 Subject: [PATCH 049/282] Fix RGB imshow with X or Y dim of size one (#1967) --- doc/whats-new.rst | 3 +++ xarray/plot/utils.py | 2 +- xarray/tests/test_plot.py | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 57ae66818d3..0a22d176435 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -64,6 +64,9 @@ Bug fixes By `Keisuke Fujii `_. - Fix kwarg `colors` clashing with auto-inferred `cmap` (:issue:`1461`) By `Deepak Cherian `_. +- Fix :py:func:`~xarray.plot.imshow` error when passed an RGB array with + size one in a spatial dimension. + By `Zac Hatfield-Dodds `_. .. _whats-new.0.10.1: diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 497705302d2..59d67ed79f1 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -296,7 +296,7 @@ def _infer_xy_labels_3d(darray, x, y, rgb): assert rgb is not None # Finally, we pick out the red slice and delegate to the 2D version: - return _infer_xy_labels(darray.isel(**{rgb: 0}).squeeze(), x, y) + return _infer_xy_labels(darray.isel(**{rgb: 0}), x, y) def _infer_xy_labels(darray, x, y, imshow=False, rgb=None): diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index d3409b6aea9..2a5eeb86bdd 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1197,6 +1197,11 @@ def test_imshow_rgb_values_in_valid_range(self): assert out.dtype == np.uint8 assert (out[..., :3] == da.values).all() # Compare without added alpha + def test_regression_rgb_imshow_dim_size_one(self): + # Regression: https://github.com/pydata/xarray/issues/1966 + da = DataArray(easy_array((1, 3, 3), start=0.0, stop=1.0)) + da.plot.imshow() + class TestFacetGrid(PlotTestCase): def setUp(self): From 33095885e6a4d7b2504ced5d9d4d34f1d6e872e2 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 8 Mar 2018 16:25:59 -0800 Subject: [PATCH 050/282] Resolve more warnings in the xarray test suite (#1964) * Resolve more warnings in the xarray test suite Currently down from 255 to 104 warnings. * Silence irrelevant warning from RasterIO * add whats-new for rasterio warnings * remove redundant import --- doc/whats-new.rst | 2 + xarray/backends/rasterio_.py | 13 +-- xarray/tests/test_dataarray.py | 2 +- xarray/tests/test_dataset.py | 2 +- xarray/tests/test_duck_array_ops.py | 123 +++++++++++++++------------- 5 files changed, 80 insertions(+), 62 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0a22d176435..d04e41ccaf8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -62,6 +62,8 @@ Bug fixes By `Stephan Hoyer `_. - Fix the precision drop after indexing datetime64 arrays (:issue:`1932`). By `Keisuke Fujii `_. +- Silenced irrelevant warnings issued by ``open_rasterio`` (:issue:`1964`). + By `Stephan Hoyer `_. - Fix kwarg `colors` clashing with auto-inferred `cmap` (:issue:`1461`) By `Deepak Cherian `_. - Fix :py:func:`~xarray.plot.imshow` error when passed an RGB array with diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index f856f9f9e13..8c0764c3ec9 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -1,7 +1,7 @@ import os -import warnings from collections import OrderedDict from distutils.version import LooseVersion +import warnings import numpy as np @@ -250,10 +250,13 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, # Is the TIF tiled? (bool) # We cast it to an int for netCDF compatibility attrs['is_tiled'] = np.uint8(riods.is_tiled) - if hasattr(riods, 'transform'): - # Affine transformation matrix (tuple of floats) - # Describes coefficients mapping pixel coordinates to CRS - attrs['transform'] = tuple(riods.transform) + with warnings.catch_warnings(): + # casting riods.transform to a tuple makes this future proof + warnings.simplefilter('ignore', FutureWarning) + if hasattr(riods, 'transform'): + # Affine transformation matrix (tuple of floats) + # Describes coefficients mapping pixel coordinates to CRS + attrs['transform'] = tuple(riods.transform) if hasattr(riods, 'nodatavals'): # The nodata values for the raster bands attrs['nodatavals'] = tuple([np.nan if nodataval is None else nodataval diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 18fc27c96ab..f42df1cbabb 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2030,7 +2030,7 @@ def test_groupby_math(self): actual = array.coords['x'] + grouped assert_identical(expected, actual) - ds = array.coords['x'].to_dataset('X') + ds = array.coords['x'].to_dataset(name='X') expected = array + ds actual = grouped + ds assert_identical(expected, actual) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index e3e2934a4fd..b4ca14d2384 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -4163,7 +4163,7 @@ def test_rolling_construct(center, window): # with fill_value ds_rolling_mean = ds_rolling.construct( 'window', stride=2, fill_value=0.0).mean('window') - assert ds_rolling_mean.isnull().sum() == 0 + assert (ds_rolling_mean.isnull().sum() == 0).to_array(dim='vars').all() assert (ds_rolling_mean['x'] == 0.0).sum() >= 0 diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 99250796d8c..2983e1991f1 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -5,6 +5,7 @@ import numpy as np import pytest from numpy import array, nan +import warnings from xarray import DataArray, concat from xarray.core.duck_array_ops import ( @@ -208,50 +209,58 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask) axis = None if aggdim is None else da.get_axis_num(aggdim) - if (LooseVersion(np.__version__) >= LooseVersion('1.13.0') and - da.dtype.kind == 'O' and skipna): - # Numpy < 1.13 does not handle object-type array. - try: - if skipna: - expected = getattr(np, 'nan{}'.format(func))(da.values, - axis=axis) - else: - expected = getattr(np, func)(da.values, axis=axis) - - actual = getattr(da, func)(skipna=skipna, dim=aggdim) - assert np.allclose(actual.values, np.array(expected), rtol=1.0e-4, - equal_nan=True) - except (TypeError, AttributeError, ZeroDivisionError): - # TODO currently, numpy does not support some methods such as - # nanmean for object dtype - pass - - # make sure the compatiblility with pandas' results. - actual = getattr(da, func)(skipna=skipna, dim=aggdim) - if func == 'var': - expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=0) - assert_allclose(actual, expected, rtol=rtol) - # also check ddof!=0 case - actual = getattr(da, func)(skipna=skipna, dim=aggdim, ddof=5) - expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=5) - assert_allclose(actual, expected, rtol=rtol) - else: - expected = series_reduce(da, func, skipna=skipna, dim=aggdim) - assert_allclose(actual, expected, rtol=rtol) - - # make sure the dtype argument - if func not in ['max', 'min']: - actual = getattr(da, func)(skipna=skipna, dim=aggdim, dtype=float) - assert actual.dtype == float - - # without nan - da = construct_dataarray(dim_num, dtype, contains_nan=False, dask=dask) - actual = getattr(da, func)(skipna=skipna) - expected = getattr(np, 'nan{}'.format(func))(da.values) - if actual.dtype == object: - assert actual.values == np.array(expected) - else: - assert np.allclose(actual.values, np.array(expected), rtol=rtol) + # TODO: remove these after resolving + # https://github.com/dask/dask/issues/3245 + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'All-NaN slice') + warnings.filterwarnings('ignore', 'invalid value encountered in') + + if (LooseVersion(np.__version__) >= LooseVersion('1.13.0') and + da.dtype.kind == 'O' and skipna): + # Numpy < 1.13 does not handle object-type array. + try: + if skipna: + expected = getattr(np, 'nan{}'.format(func))(da.values, + axis=axis) + else: + expected = getattr(np, func)(da.values, axis=axis) + + actual = getattr(da, func)(skipna=skipna, dim=aggdim) + assert np.allclose(actual.values, np.array(expected), + rtol=1.0e-4, equal_nan=True) + except (TypeError, AttributeError, ZeroDivisionError): + # TODO currently, numpy does not support some methods such as + # nanmean for object dtype + pass + + # make sure the compatiblility with pandas' results. + actual = getattr(da, func)(skipna=skipna, dim=aggdim) + if func == 'var': + expected = series_reduce(da, func, skipna=skipna, dim=aggdim, + ddof=0) + assert_allclose(actual, expected, rtol=rtol) + # also check ddof!=0 case + actual = getattr(da, func)(skipna=skipna, dim=aggdim, ddof=5) + expected = series_reduce(da, func, skipna=skipna, dim=aggdim, + ddof=5) + assert_allclose(actual, expected, rtol=rtol) + else: + expected = series_reduce(da, func, skipna=skipna, dim=aggdim) + assert_allclose(actual, expected, rtol=rtol) + + # make sure the dtype argument + if func not in ['max', 'min']: + actual = getattr(da, func)(skipna=skipna, dim=aggdim, dtype=float) + assert actual.dtype == float + + # without nan + da = construct_dataarray(dim_num, dtype, contains_nan=False, dask=dask) + actual = getattr(da, func)(skipna=skipna) + expected = getattr(np, 'nan{}'.format(func))(da.values) + if actual.dtype == object: + assert actual.values == np.array(expected) + else: + assert np.allclose(actual.values, np.array(expected), rtol=rtol) @pytest.mark.parametrize('dim_num', [1, 2]) @@ -280,17 +289,21 @@ def test_argmin_max(dim_num, dtype, contains_nan, dask, func, skipna, aggdim): da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, dask=dask) - if aggdim == 'y' and contains_nan and skipna: - with pytest.raises(ValueError): - actual = da.isel(**{ - aggdim: getattr(da, 'arg' + func)(dim=aggdim, - skipna=skipna).compute()}) - return - - actual = da.isel(**{aggdim: getattr(da, 'arg' + func) - (dim=aggdim, skipna=skipna).compute()}) - expected = getattr(da, func)(dim=aggdim, skipna=skipna) - assert_allclose(actual.drop(actual.coords), expected.drop(expected.coords)) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'All-NaN slice') + + if aggdim == 'y' and contains_nan and skipna: + with pytest.raises(ValueError): + actual = da.isel(**{ + aggdim: getattr(da, 'arg' + func)( + dim=aggdim, skipna=skipna).compute()}) + return + + actual = da.isel(**{aggdim: getattr(da, 'arg' + func) + (dim=aggdim, skipna=skipna).compute()}) + expected = getattr(da, func)(dim=aggdim, skipna=skipna) + assert_allclose(actual.drop(actual.coords), + expected.drop(expected.coords)) def test_argmin_max_error(): From 8c6a28435282b1fd988e3984ecec539de94b10ca Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 9 Mar 2018 11:22:19 -0800 Subject: [PATCH 051/282] Use conda-forge netcdftime wherever netcdf4 was tested (#1933) * use stand-alone netcdftime from conda-forge * remove temporary netcdftime development build * more docs, bump for tests * update docs a bit --- .travis.yml | 4 ---- ci/requirements-py27-cdat+iris+pynio.yml | 1 + ci/requirements-py27-windows.yml | 1 + ci/requirements-py35.yml | 1 + ci/requirements-py36-bottleneck-dev.yml | 1 + ci/requirements-py36-condaforge-rc.yml | 1 + ci/requirements-py36-dask-dev.yml | 1 + ci/requirements-py36-netcdf4-dev.yml | 1 + ci/requirements-py36-netcdftime-dev.yml | 13 ------------- ci/requirements-py36-pandas-dev.yml | 1 + ci/requirements-py36-pynio-dev.yml | 1 + ci/requirements-py36-rasterio1.0alpha.yml | 1 + ci/requirements-py36-windows.yml | 1 + ci/requirements-py36-zarr-dev.yml | 1 + ci/requirements-py36.yml | 1 + doc/installing.rst | 5 +++-- doc/time-series.rst | 11 +++++++++++ doc/whats-new.rst | 14 ++++++++++++-- 18 files changed, 39 insertions(+), 21 deletions(-) delete mode 100644 ci/requirements-py36-netcdftime-dev.yml diff --git a/.travis.yml b/.travis.yml index 70c0a63ae08..ee8ffcc4d5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,8 +45,6 @@ matrix: env: CONDA_ENV=py36-rasterio1.0alpha - python: 3.6 env: CONDA_ENV=py36-zarr-dev - - python: 3.6 - env: CONDA_ENV=py36-netcdftime-dev - python: 3.5 env: CONDA_ENV=docs allow_failures: @@ -75,8 +73,6 @@ matrix: env: CONDA_ENV=py36-rasterio1.0alpha - python: 3.6 env: CONDA_ENV=py36-zarr-dev - - python: 3.6 - env: CONDA_ENV=py36-netcdftime-dev before_install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then diff --git a/ci/requirements-py27-cdat+iris+pynio.yml b/ci/requirements-py27-cdat+iris+pynio.yml index 6c7e3b87318..119d4eb0ffc 100644 --- a/ci/requirements-py27-cdat+iris+pynio.yml +++ b/ci/requirements-py27-cdat+iris+pynio.yml @@ -11,6 +11,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - numpy - pandas - pathlib2 diff --git a/ci/requirements-py27-windows.yml b/ci/requirements-py27-windows.yml index a39b24b887c..32aa0da0e96 100644 --- a/ci/requirements-py27-windows.yml +++ b/ci/requirements-py27-windows.yml @@ -9,6 +9,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pathlib2 - pytest - flake8 diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index 6f9ae2490b9..10c0eaead95 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -9,6 +9,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-bottleneck-dev.yml b/ci/requirements-py36-bottleneck-dev.yml index 571a2e1294f..2d7a715ee44 100644 --- a/ci/requirements-py36-bottleneck-dev.yml +++ b/ci/requirements-py36-bottleneck-dev.yml @@ -9,6 +9,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-condaforge-rc.yml b/ci/requirements-py36-condaforge-rc.yml index 6519d0d0f47..7e61514b877 100644 --- a/ci/requirements-py36-condaforge-rc.yml +++ b/ci/requirements-py36-condaforge-rc.yml @@ -10,6 +10,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-dask-dev.yml b/ci/requirements-py36-dask-dev.yml index ae359a13356..4bffce496f9 100644 --- a/ci/requirements-py36-dask-dev.yml +++ b/ci/requirements-py36-dask-dev.yml @@ -7,6 +7,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-netcdf4-dev.yml b/ci/requirements-py36-netcdf4-dev.yml index 2daa02756bb..7d046743e7e 100644 --- a/ci/requirements-py36-netcdf4-dev.yml +++ b/ci/requirements-py36-netcdf4-dev.yml @@ -19,3 +19,4 @@ dependencies: - coveralls - pytest-cov - git+https://github.com/Unidata/netcdf4-python.git + - git+https://github.com/Unidata/netcdftime.git diff --git a/ci/requirements-py36-netcdftime-dev.yml b/ci/requirements-py36-netcdftime-dev.yml deleted file mode 100644 index 5c2193474b4..00000000000 --- a/ci/requirements-py36-netcdftime-dev.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: test_env -channels: - - conda-forge -dependencies: - - python=3.6 - - pytest - - flake8 - - numpy - - pandas - - netcdftime - - pip: - - coveralls - - pytest-cov diff --git a/ci/requirements-py36-pandas-dev.yml b/ci/requirements-py36-pandas-dev.yml index fe4eb226204..6ecd1c47f0d 100644 --- a/ci/requirements-py36-pandas-dev.yml +++ b/ci/requirements-py36-pandas-dev.yml @@ -10,6 +10,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-pynio-dev.yml b/ci/requirements-py36-pynio-dev.yml index e19c6537c68..4667a66444b 100644 --- a/ci/requirements-py36-pynio-dev.yml +++ b/ci/requirements-py36-pynio-dev.yml @@ -10,6 +10,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pynio=dev - pytest - numpy diff --git a/ci/requirements-py36-rasterio1.0alpha.yml b/ci/requirements-py36-rasterio1.0alpha.yml index 3c32ebb0e43..8a2942a4d08 100644 --- a/ci/requirements-py36-rasterio1.0alpha.yml +++ b/ci/requirements-py36-rasterio1.0alpha.yml @@ -10,6 +10,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - numpy - pandas diff --git a/ci/requirements-py36-windows.yml b/ci/requirements-py36-windows.yml index ea366bd04f7..228ce285787 100644 --- a/ci/requirements-py36-windows.yml +++ b/ci/requirements-py36-windows.yml @@ -9,6 +9,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - numpy - pandas diff --git a/ci/requirements-py36-zarr-dev.yml b/ci/requirements-py36-zarr-dev.yml index 9be522882c5..b5991ba4403 100644 --- a/ci/requirements-py36-zarr-dev.yml +++ b/ci/requirements-py36-zarr-dev.yml @@ -14,6 +14,7 @@ dependencies: - seaborn - toolz - bottleneck + - netcdftime - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index cc02d6e92bf..2788221ee74 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -9,6 +9,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - netcdftime - pytest - flake8 - numpy diff --git a/doc/installing.rst b/doc/installing.rst index 8be025665e2..df443cbfed5 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -25,8 +25,9 @@ For netCDF and IO - `pynio `__: for reading GRIB and other geoscience specific file formats - `zarr `__: for chunked, compressed, N-dimensional arrays. -- `netcdftime `__: recommended if you - want to encode/decode datetimes for non-standard calendars. +- `netcdftime `__: recommended if you + want to encode/decode datetimes for non-standard calendars or dates before + year 1678 or after year 2262. For accelerating xarray ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/time-series.rst b/doc/time-series.rst index 4d9a995051a..acc10220f27 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -47,6 +47,17 @@ attribute like ``'days since 2000-01-01'``). .. _CF conventions: http://cfconventions.org +.. note:: + + When decoding/encoding datetimes for non-standard calendars or for dates + before year 1678 or after year 2262, xarray uses the `netcdftime`_ library. + ``netcdftime`` was previously packaged with the ``netcdf4-python`` package but + is now distributed separately. ``netcdftime`` is an + :ref:`optional dependency` of xarray. + +.. _netcdftime: https://unidata.github.io/netcdftime + + You can manual decode arrays in this form by passing a dataset to :py:func:`~xarray.decode_cf`: diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d04e41ccaf8..5b0d8ad522f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -154,15 +154,25 @@ Enhancements By `Stephan Hoyer `_. - Fix ``axis`` keyword ignored when applying ``np.squeeze`` to ``DataArray`` (:issue:`1487`). By `Florian Pinault `_. -- Add ``netcdftime`` as an optional dependency of xarray. This allows for +- ``netcdf4-python`` has moved the its time handling in the ``netcdftime`` module to + a standalone package (`netcdftime`_). As such, xarray now considers `netcdftime`_ + an optional dependency. One benefit of this change is that it allows for encoding/decoding of datetimes with non-standard calendars without the - netCDF4 dependency (:issue:`1084`). + ``netcdf4-python`` dependency (:issue:`1084`). By `Joe Hamman `_. .. _Zarr: http://zarr.readthedocs.io/ .. _Iris: http://scitools.org.uk/iris +.. _netcdftime: https://unidata.github.io/netcdftime + +**New functions/methods** + +- New :py:meth:`~xarray.DataArray.rank` on arrays and datasets. Requires + bottleneck (:issue:`1731`). + By `0x0L `_. + Bug fixes ~~~~~~~~~ - Rolling aggregation with ``center=True`` option now gives the same result From 2f590f7d7f34c7dfddea4d1d4f8877bca081b601 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 10 Mar 2018 07:43:17 -0800 Subject: [PATCH 052/282] fix distributed writes (#1793) * distributed tests that write dask arrays * Change zarr test to synchronous API * initial go at __setitem__ on array wrappers * fixes for scipy * cleanup after merging with upstream/master * needless duplication of tests to work around pytest bug * use netcdf_variable instead of get_array() * use synchronous dask.distributed test harness * cleanup tests * per scheduler locks and autoclose behavior for writes * HDF5_LOCK and CombinedLock * integration test for distributed locks * more tests and set isopen to false when pickling * Fixing style errors. * ds property on DataStorePickleMixin * stickler-ci * compat fixes for other backends * HDF5_USE_FILE_LOCKING = False in test_distributed * style fix * update tests to only expect netcdf4 to work, docstrings, and some cleanup in to_netcdf * Fixing style errors. * fix imports after merge * fix more import bugs * update docs * fix for pynio * cleanup locks and use pytest monkeypatch for environment variable * fix failing test using combined lock --- doc/dask.rst | 8 ++ doc/io.rst | 6 +- doc/whats-new.rst | 7 ++ xarray/backends/api.py | 35 +++++++- xarray/backends/common.py | 135 ++++++++++++++++++++++++++----- xarray/backends/h5netcdf_.py | 14 ++-- xarray/backends/netCDF4_.py | 23 ++++-- xarray/backends/pynio_.py | 8 +- xarray/backends/scipy_.py | 30 +++++-- xarray/backends/zarr.py | 5 ++ xarray/core/combine.py | 2 +- xarray/tests/test_distributed.py | 122 ++++++++++++++++++++++++---- 12 files changed, 331 insertions(+), 64 deletions(-) diff --git a/doc/dask.rst b/doc/dask.rst index 824f30aba4f..8fc0f655023 100644 --- a/doc/dask.rst +++ b/doc/dask.rst @@ -100,6 +100,14 @@ Once you've manipulated a dask array, you can still write a dataset too big to fit into memory back to disk by using :py:meth:`~xarray.Dataset.to_netcdf` in the usual way. +.. note:: + + When using dask's distributed scheduler to write NETCDF4 files, + it may be necessary to set the environment variable `HDF5_USE_FILE_LOCKING=FALSE` + to avoid competing locks within the HDF5 SWMR file locking scheme. Note that + writing netCDF files with dask's distributed scheduler is only supported for + the `netcdf4` backend. + A dataset can also be converted to a dask DataFrame using :py:meth:`~xarray.Dataset.to_dask_dataframe`. .. ipython:: python diff --git a/doc/io.rst b/doc/io.rst index c177496f6f2..c14e1516b38 100644 --- a/doc/io.rst +++ b/doc/io.rst @@ -672,9 +672,9 @@ files into a single Dataset by making use of :py:func:`~xarray.concat`. .. note:: - Version 0.5 includes support for manipulating datasets that - don't fit into memory with dask_. If you have dask installed, you can open - multiple files simultaneously using :py:func:`~xarray.open_mfdataset`:: + Xarray includes support for manipulating datasets that don't fit into memory + with dask_. If you have dask installed, you can open multiple files + simultaneously using :py:func:`~xarray.open_mfdataset`:: xr.open_mfdataset('my/files/*.nc') diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 5b0d8ad522f..963a0454f88 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -38,6 +38,13 @@ Documentation Enhancements ~~~~~~~~~~~~ +- Support for writing xarray datasets to netCDF files (netcdf4 backend only) + when using the `dask.distributed `_ + scheduler (:issue:`1464`). + By `Joe Hamman `_. + + +- Fixed to_netcdf when using dask distributed - Support lazy vectorized-indexing. After this change, flexible indexing such as orthogonal/vectorized indexing, becomes possible for all the backend arrays. Also, lazy ``transpose`` is now also supported. (:issue:`1897`) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 9d0b95c8c81..a22356f66b0 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -12,7 +12,8 @@ from ..core.combine import auto_combine from ..core.pycompat import basestring, path_type from ..core.utils import close_on_error, is_remote_uri -from .common import GLOBAL_LOCK, ArrayWriter +from .common import ( + HDF5_LOCK, ArrayWriter, CombinedLock, get_scheduler, get_scheduler_lock) DATAARRAY_NAME = '__xarray_dataarray_name__' DATAARRAY_VARIABLE = '__xarray_dataarray_variable__' @@ -64,9 +65,9 @@ def _default_lock(filename, engine): else: # TODO: identify netcdf3 files and don't use the global lock # for them - lock = GLOBAL_LOCK + lock = HDF5_LOCK elif engine in {'h5netcdf', 'pynio'}: - lock = GLOBAL_LOCK + lock = HDF5_LOCK else: lock = False return lock @@ -129,6 +130,20 @@ def _protect_dataset_variables_inplace(dataset, cache): variable.data = data +def _get_lock(engine, scheduler, format, path_or_file): + """ Get the lock(s) that apply to a particular scheduler/engine/format""" + + locks = [] + if format in ['NETCDF4', None] and engine in ['h5netcdf', 'netcdf4']: + locks.append(HDF5_LOCK) + locks.append(get_scheduler_lock(scheduler, path_or_file)) + + # When we have more than one lock, use the CombinedLock wrapper class + lock = CombinedLock(locks) if len(locks) > 1 else locks[0] + + return lock + + def open_dataset(filename_or_obj, group=None, decode_cf=True, mask_and_scale=True, decode_times=True, autoclose=False, concat_characters=True, decode_coords=True, engine=None, @@ -620,8 +635,20 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, # if a writer is provided, store asynchronously sync = writer is None + # handle scheduler specific logic + scheduler = get_scheduler() + if (dataset.chunks and scheduler in ['distributed', 'multiprocessing'] and + engine != 'netcdf4'): + raise NotImplementedError("Writing netCDF files with the %s backend " + "is not currently supported with dask's %s " + "scheduler" % (engine, scheduler)) + lock = _get_lock(engine, scheduler, format, path_or_file) + autoclose = (dataset.chunks and + scheduler in ['distributed', 'multiprocessing']) + target = path_or_file if path_or_file is not None else BytesIO() - store = store_open(target, mode, format, group, writer) + store = store_open(target, mode, format, group, writer, + autoclose=autoclose, lock=lock) if unlimited_dims is None: unlimited_dims = dataset.encoding.get('unlimited_dims', None) diff --git a/xarray/backends/common.py b/xarray/backends/common.py index d91cedbbda3..c46f9d5b552 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -2,6 +2,8 @@ import contextlib import logging +import multiprocessing +import threading import time import traceback import warnings @@ -14,11 +16,12 @@ from ..core.pycompat import dask_array_type, iteritems from ..core.utils import FrozenOrderedDict, NdimSizeLenMixin +# Import default lock try: - from dask.utils import SerializableLock as Lock + from dask.utils import SerializableLock + HDF5_LOCK = SerializableLock() except ImportError: - from threading import Lock - + HDF5_LOCK = threading.Lock() # Create a logger object, but don't add any handlers. Leave that to user code. logger = logging.getLogger(__name__) @@ -27,8 +30,54 @@ NONE_VAR_NAME = '__values__' -# dask.utils.SerializableLock if available, otherwise just a threading.Lock -GLOBAL_LOCK = Lock() +def get_scheduler(get=None, collection=None): + """ Determine the dask scheduler that is being used. + + None is returned if not dask scheduler is active. + + See also + -------- + dask.utils.effective_get + """ + try: + from dask.utils import effective_get + actual_get = effective_get(get, collection) + try: + from dask.distributed import Client + if isinstance(actual_get.__self__, Client): + return 'distributed' + except (ImportError, AttributeError): + try: + import dask.multiprocessing + if actual_get == dask.multiprocessing.get: + return 'multiprocessing' + else: + return 'threaded' + except ImportError: + return 'threaded' + except ImportError: + return None + + +def get_scheduler_lock(scheduler, path_or_file=None): + """ Get the appropriate lock for a certain situation based onthe dask + scheduler used. + + See Also + -------- + dask.utils.get_scheduler_lock + """ + + if scheduler == 'distributed': + from dask.distributed import Lock + return Lock(path_or_file) + elif scheduler == 'multiprocessing': + return multiprocessing.Lock() + elif scheduler == 'threaded': + from dask.utils import SerializableLock + return SerializableLock() + else: + return threading.Lock() def _encode_variable_name(name): @@ -77,6 +126,39 @@ def robust_getitem(array, key, catch=Exception, max_retries=6, time.sleep(1e-3 * next_delay) +class CombinedLock(object): + """A combination of multiple locks. + + Like a locked door, a CombinedLock is locked if any of its constituent + locks are locked. + """ + + def __init__(self, locks): + self.locks = tuple(set(locks)) # remove duplicates + + def acquire(self, *args): + return all(lock.acquire(*args) for lock in self.locks) + + def release(self, *args): + for lock in self.locks: + lock.release(*args) + + def __enter__(self): + for lock in self.locks: + lock.__enter__() + + def __exit__(self, *args): + for lock in self.locks: + lock.__exit__(*args) + + @property + def locked(self): + return any(lock.locked for lock in self.locks) + + def __repr__(self): + return "CombinedLock(%r)" % list(self.locks) + + class BackendArray(NdimSizeLenMixin, indexing.ExplicitlyIndexed): def __array__(self, dtype=None): @@ -85,7 +167,9 @@ def __array__(self, dtype=None): class AbstractDataStore(Mapping): - _autoclose = False + _autoclose = None + _ds = None + _isopen = False def __iter__(self): return iter(self.variables) @@ -168,7 +252,7 @@ def __exit__(self, exception_type, exception_value, traceback): class ArrayWriter(object): - def __init__(self, lock=GLOBAL_LOCK): + def __init__(self, lock=HDF5_LOCK): self.sources = [] self.targets = [] self.lock = lock @@ -178,11 +262,7 @@ def add(self, source, target): self.sources.append(source) self.targets.append(target) else: - try: - target[...] = source - except TypeError: - # workaround for GH: scipy/scipy#6880 - target[:] = source + target[...] = source def sync(self): if self.sources: @@ -193,9 +273,9 @@ def sync(self): class AbstractWritableDataStore(AbstractDataStore): - def __init__(self, writer=None): + def __init__(self, writer=None, lock=HDF5_LOCK): if writer is None: - writer = ArrayWriter() + writer = ArrayWriter(lock=lock) self.writer = writer def encode(self, variables, attributes): @@ -239,6 +319,9 @@ def set_variable(self, k, v): # pragma: no cover raise NotImplementedError def sync(self): + if self._isopen and self._autoclose: + # datastore will be reopened during write + self.close() self.writer.sync() def store_dataset(self, dataset): @@ -373,7 +456,8 @@ class DataStorePickleMixin(object): def __getstate__(self): state = self.__dict__.copy() - del state['ds'] + del state['_ds'] + del state['_isopen'] if self._mode == 'w': # file has already been created, don't override when restoring state['_mode'] = 'a' @@ -381,19 +465,32 @@ def __getstate__(self): def __setstate__(self, state): self.__dict__.update(state) - self.ds = self._opener(mode=self._mode) + self._ds = None + self._isopen = False + + @property + def ds(self): + if self._ds is not None and self._isopen: + return self._ds + ds = self._opener(mode=self._mode) + self._isopen = True + return ds @contextlib.contextmanager - def ensure_open(self, autoclose): + def ensure_open(self, autoclose=None): """ Helper function to make sure datasets are closed and opened at appropriate times to avoid too many open file errors. Use requires `autoclose=True` argument to `open_mfdataset`. """ - if self._autoclose and not self._isopen: + + if autoclose is None: + autoclose = self._autoclose + + if not self._isopen: try: - self.ds = self._opener() + self._ds = self._opener() self._isopen = True yield finally: diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 1d166f05eb1..7beda03308e 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -8,7 +8,8 @@ from ..core import indexing from ..core.pycompat import OrderedDict, bytes_type, iteritems, unicode_type from ..core.utils import FrozenOrderedDict, close_on_error -from .common import DataStorePickleMixin, WritableCFDataStore, find_root +from .common import ( + HDF5_LOCK, DataStorePickleMixin, WritableCFDataStore, find_root) from .netCDF4_ import ( BaseNetCDF4Array, _encode_nc4_variable, _extract_nc4_variable_encoding, _get_datatype, _nc4_group) @@ -68,12 +69,12 @@ class H5NetCDFStore(WritableCFDataStore, DataStorePickleMixin): """ def __init__(self, filename, mode='r', format=None, group=None, - writer=None, autoclose=False): + writer=None, autoclose=False, lock=HDF5_LOCK): if format not in [None, 'NETCDF4']: raise ValueError('invalid format for h5netcdf backend') opener = functools.partial(_open_h5netcdf_group, filename, mode=mode, group=group) - self.ds = opener() + self._ds = opener() if autoclose: raise NotImplementedError('autoclose=True is not implemented ' 'for the h5netcdf backend pending ' @@ -85,7 +86,7 @@ def __init__(self, filename, mode='r', format=None, group=None, self._opener = opener self._filename = filename self._mode = mode - super(H5NetCDFStore, self).__init__(writer) + super(H5NetCDFStore, self).__init__(writer, lock=lock) def open_store_variable(self, name, var): with self.ensure_open(autoclose=False): @@ -177,7 +178,10 @@ def prepare_variable(self, name, variable, check_encoding=False, for k, v in iteritems(attrs): nc4_var.setncattr(k, v) - return nc4_var, variable.data + + target = H5NetCDFArrayWrapper(name, self) + + return target, variable.data def sync(self): with self.ensure_open(autoclose=True): diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 4903e9a98f2..01d1a4de5f5 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -13,8 +13,8 @@ from ..core.pycompat import PY3, OrderedDict, basestring, iteritems, suppress from ..core.utils import FrozenOrderedDict, close_on_error, is_remote_uri from .common import ( - BackendArray, DataStorePickleMixin, WritableCFDataStore, find_root, - robust_getitem) + HDF5_LOCK, BackendArray, DataStorePickleMixin, WritableCFDataStore, + find_root, robust_getitem) from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable # This lookup table maps from dtype.byteorder to a readable endian @@ -41,6 +41,11 @@ def __init__(self, variable_name, datastore): dtype = np.dtype('O') self.dtype = dtype + def __setitem__(self, key, value): + with self.datastore.ensure_open(autoclose=True): + data = self.get_array() + data[key] = value + def get_array(self): self.datastore.assert_open() return self.datastore.ds.variables[self.variable_name] @@ -231,14 +236,14 @@ class NetCDF4DataStore(WritableCFDataStore, DataStorePickleMixin): """ def __init__(self, netcdf4_dataset, mode='r', writer=None, opener=None, - autoclose=False): + autoclose=False, lock=HDF5_LOCK): if autoclose and opener is None: raise ValueError('autoclose requires an opener') _disable_auto_decode_group(netcdf4_dataset) - self.ds = netcdf4_dataset + self._ds = netcdf4_dataset self._autoclose = autoclose self._isopen = True self.format = self.ds.data_model @@ -249,12 +254,12 @@ def __init__(self, netcdf4_dataset, mode='r', writer=None, opener=None, self._opener = functools.partial(opener, mode=self._mode) else: self._opener = opener - super(NetCDF4DataStore, self).__init__(writer) + super(NetCDF4DataStore, self).__init__(writer, lock=lock) @classmethod def open(cls, filename, mode='r', format='NETCDF4', group=None, writer=None, clobber=True, diskless=False, persist=False, - autoclose=False): + autoclose=False, lock=HDF5_LOCK): import netCDF4 as nc4 if (len(filename) == 88 and LooseVersion(nc4.__version__) < "1.3.1"): @@ -274,7 +279,7 @@ def open(cls, filename, mode='r', format='NETCDF4', group=None, format=format) ds = opener() return cls(ds, mode=mode, writer=writer, opener=opener, - autoclose=autoclose) + autoclose=autoclose, lock=lock) def open_store_variable(self, name, var): with self.ensure_open(autoclose=False): @@ -399,7 +404,9 @@ def prepare_variable(self, name, variable, check_encoding=False, # OrderedDict as the input to setncatts nc4_var.setncattr(k, v) - return nc4_var, variable.data + target = NetCDF4ArrayWrapper(name, self) + + return target, variable.data def sync(self): with self.ensure_open(autoclose=True): diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 95226e453b4..3c638b6b057 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -46,14 +46,14 @@ class NioDataStore(AbstractDataStore, DataStorePickleMixin): def __init__(self, filename, mode='r', autoclose=False): import Nio opener = functools.partial(Nio.open_file, filename, mode=mode) - self.ds = opener() - # xarray provides its own support for FillValue, - # so turn off PyNIO's support for the same. - self.ds.set_option('MaskedArrayMode', 'MaskedNever') + self._ds = opener() self._autoclose = autoclose self._isopen = True self._opener = opener self._mode = mode + # xarray provides its own support for FillValue, + # so turn off PyNIO's support for the same. + self.ds.set_option('MaskedArrayMode', 'MaskedNever') def open_store_variable(self, name, var): data = indexing.LazilyOuterIndexedArray(NioArrayWrapper(name, self)) diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index a0765fe27bd..ee2c0fbf106 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -2,6 +2,7 @@ import functools import warnings +from distutils.version import LooseVersion from io import BytesIO import numpy as np @@ -53,6 +54,18 @@ def __getitem__(self, key): copy = self.datastore.ds.use_mmap return np.array(data, dtype=self.dtype, copy=copy) + def __setitem__(self, key, value): + with self.datastore.ensure_open(autoclose=True): + data = self.datastore.ds.variables[self.variable_name] + try: + data[key] = value + except TypeError: + if key is Ellipsis: + # workaround for GH: scipy/scipy#6880 + data[:] = value + else: + raise + def _open_scipy_netcdf(filename, mode, mmap, version): import scipy.io @@ -103,11 +116,12 @@ class ScipyDataStore(WritableCFDataStore, DataStorePickleMixin): """ def __init__(self, filename_or_obj, mode='r', format=None, group=None, - writer=None, mmap=None, autoclose=False): + writer=None, mmap=None, autoclose=False, lock=None): import scipy import scipy.io - if mode != 'r' and scipy.__version__ < '0.13': # pragma: no cover + if (mode != 'r' and + scipy.__version__ < LooseVersion('0.13')): # pragma: no cover warnings.warn('scipy %s detected; ' 'the minimal recommended version is 0.13. ' 'Older version of this library do not reliably ' @@ -129,13 +143,13 @@ def __init__(self, filename_or_obj, mode='r', format=None, group=None, opener = functools.partial(_open_scipy_netcdf, filename=filename_or_obj, mode=mode, mmap=mmap, version=version) - self.ds = opener() + self._ds = opener() self._autoclose = autoclose self._isopen = True self._opener = opener self._mode = mode - super(ScipyDataStore, self).__init__(writer) + super(ScipyDataStore, self).__init__(writer, lock=lock) def open_store_variable(self, name, var): with self.ensure_open(autoclose=False): @@ -200,7 +214,10 @@ def prepare_variable(self, name, variable, check_encoding=False, for k, v in iteritems(variable.attrs): self._validate_attr_key(k) setattr(scipy_var, k, v) - return scipy_var, data + + target = ScipyArrayWrapper(name, self) + + return target, data def sync(self): with self.ensure_open(autoclose=True): @@ -221,4 +238,5 @@ def __setstate__(self, state): # seek to the start of the file so scipy can read it filename.seek(0) super(ScipyDataStore, self).__setstate__(state) - self._isopen = True + self._ds = None + self._isopen = False diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 8797e3104a1..71ce965f368 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -341,6 +341,8 @@ def prepare_variable(self, name, variable, check_encoding=False, fill_value = _ensure_valid_fill_value(attrs.pop('_FillValue', None), dtype) + if variable.encoding == {'_FillValue': None} and fill_value is None: + variable.encoding = {} encoding = _extract_zarr_variable_encoding( variable, raise_on_invalid=check_encoding) @@ -361,6 +363,9 @@ def store(self, variables, attributes, *args, **kwargs): AbstractWritableDataStore.store(self, variables, attributes, *args, **kwargs) + def sync(self): + self.writer.sync() + def open_zarr(store, group=None, synchronizer=None, auto_chunk=True, decode_cf=True, mask_and_scale=True, decode_times=True, diff --git a/xarray/core/combine.py b/xarray/core/combine.py index 149009689e9..8c1c58e9a40 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -8,8 +8,8 @@ from .alignment import align from .merge import merge from .pycompat import OrderedDict, basestring, iteritems -from .variable import IndexVariable, Variable, as_variable from .variable import concat as concat_vars +from .variable import IndexVariable, Variable, as_variable def concat(objs, dim=None, data_vars='all', coords='different', diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 0d060069477..0ac03327494 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -1,6 +1,9 @@ """ isort:skip_file """ - +from __future__ import absolute_import, division, print_function +import os import sys +import pickle +import tempfile import pytest @@ -8,6 +11,7 @@ distributed = pytest.importorskip('distributed') # isort:skip from dask import array +from dask.distributed import Client, Lock from distributed.utils_test import cluster, gen_cluster from distributed.utils_test import loop # flake8: noqa from distributed.client import futures_of @@ -15,9 +19,11 @@ import xarray as xr from xarray.tests.test_backends import ON_WINDOWS, create_tmp_file from xarray.tests.test_dataset import create_test_data +from xarray.backends.common import HDF5_LOCK, CombinedLock from . import ( - assert_allclose, has_h5netcdf, has_netCDF4, has_scipy, requires_zarr) + assert_allclose, has_h5netcdf, has_netCDF4, has_scipy, requires_zarr, + raises_regex) # this is to stop isort throwing errors. May have been easier to just use # `isort:skip` in retrospect @@ -34,29 +40,95 @@ if has_h5netcdf: ENGINES.append('h5netcdf') +NC_FORMATS = {'netcdf4': ['NETCDF3_CLASSIC', 'NETCDF3_64BIT_OFFSET', + 'NETCDF3_64BIT_DATA', 'NETCDF4_CLASSIC', 'NETCDF4'], + 'scipy': ['NETCDF3_CLASSIC', 'NETCDF3_64BIT'], + 'h5netcdf': ['NETCDF4']} +TEST_FORMATS = ['NETCDF3_CLASSIC', 'NETCDF4_CLASSIC', 'NETCDF4'] + + +@pytest.mark.xfail(sys.platform == 'win32', + reason='https://github.com/pydata/xarray/issues/1738') +@pytest.mark.parametrize('engine', ['netcdf4']) +@pytest.mark.parametrize('autoclose', [True, False]) +@pytest.mark.parametrize('nc_format', TEST_FORMATS) +def test_dask_distributed_netcdf_roundtrip(monkeypatch, loop, + engine, autoclose, nc_format): + + monkeypatch.setenv('HDF5_USE_FILE_LOCKING', 'FALSE') + + chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} + + with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: + + original = create_test_data().chunk(chunks) + original.to_netcdf(filename, engine=engine, format=nc_format) + + with xr.open_dataset(filename, + chunks=chunks, + engine=engine, + autoclose=autoclose) as restored: + assert isinstance(restored.var1.data, da.Array) + computed = restored.compute() + assert_allclose(original, computed) + @pytest.mark.xfail(sys.platform == 'win32', reason='https://github.com/pydata/xarray/issues/1738') @pytest.mark.parametrize('engine', ENGINES) -def test_dask_distributed_netcdf_integration_test(loop, engine): - with cluster() as (s, _): - with distributed.Client(s['address'], loop=loop): - original = create_test_data() - with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: - original.to_netcdf(filename, engine=engine) - with xr.open_dataset( - filename, chunks=3, engine=engine) as restored: +@pytest.mark.parametrize('autoclose', [True, False]) +@pytest.mark.parametrize('nc_format', TEST_FORMATS) +def test_dask_distributed_read_netcdf_integration_test(loop, engine, autoclose, + nc_format): + + if engine == 'h5netcdf' and autoclose: + pytest.skip('h5netcdf does not support autoclose') + + if nc_format not in NC_FORMATS[engine]: + pytest.skip('invalid format for engine') + + chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} + + with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: + + original = create_test_data() + original.to_netcdf(filename, engine=engine, format=nc_format) + + with xr.open_dataset(filename, + chunks=chunks, + engine=engine, + autoclose=autoclose) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed) +@pytest.mark.parametrize('engine', ['h5netcdf', 'scipy']) +def test_dask_distributed_netcdf_integration_test_not_implemented(loop, engine): + chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} + + with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: + + original = create_test_data().chunk(chunks) + + with raises_regex(NotImplementedError, 'distributed'): + original.to_netcdf(filename, engine=engine) + + @requires_zarr def test_dask_distributed_zarr_integration_test(loop): - with cluster() as (s, _): - with distributed.Client(s['address'], loop=loop): - original = create_test_data() - with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: + chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5} + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: + original = create_test_data().chunk(chunks) + with create_tmp_file(allow_cleanup_failure=ON_WINDOWS, + suffix='.zarr') as filename: original.to_zarr(filename) with xr.open_zarr(filename) as restored: assert isinstance(restored.var1.data, da.Array) @@ -92,3 +164,25 @@ def test_async(c, s, a, b): assert_allclose(x + 10, w) assert s.tasks + + +def test_hdf5_lock(): + assert isinstance(HDF5_LOCK, dask.utils.SerializableLock) + + +@gen_cluster(client=True) +def test_serializable_locks(c, s, a, b): + def f(x, lock=None): + with lock: + return x + 1 + + # note, the creation of Lock needs to be done inside a cluster + for lock in [HDF5_LOCK, Lock(), Lock('filename.nc'), + CombinedLock([HDF5_LOCK]), + CombinedLock([HDF5_LOCK, Lock('filename.nc')])]: + + futures = c.map(f, list(range(10)), lock=lock) + yield c.gather(futures) + + lock2 = pickle.loads(pickle.dumps(lock)) + assert type(lock) == type(lock2) From aa83d0ec5a0da9e8880d3194864ff212d5990d6b Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 11 Mar 2018 22:37:02 -0700 Subject: [PATCH 053/282] quick fix for failing zarr test (#1980) --- xarray/tests/test_backends.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 21152829096..5b9bb2a0506 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1340,8 +1340,10 @@ def test_compressor_encoding(self): import zarr blosc_comp = zarr.Blosc(cname='zstd', clevel=3, shuffle=2) save_kwargs = dict(encoding={'var1': {'compressor': blosc_comp}}) - with self.roundtrip(original, save_kwargs=save_kwargs) as actual: - assert repr(actual.var1.encoding['compressor']) == repr(blosc_comp) + with self.roundtrip(original, save_kwargs=save_kwargs) as ds: + actual = ds['var1'].encoding['compressor'] + # get_config returns a dictionary of compressor attributes + assert actual.get_config() == blosc_comp.get_config() def test_group(self): original = create_test_data() From 8271dffc63ec2b12fa81b11381981c9f900449e7 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Mon, 12 Mar 2018 15:42:07 +0900 Subject: [PATCH 054/282] einsum for xarray (#1968) * einsum for xarray * whats new * Support dask for xr.dot. * flake8. Add some error messages. * fix for sticker-ci * Use counter * Always allow dims=None for xr.dot. * Simplify logic. More comments. * Support variable in xr.dot * bug fix due to the undefined order of set * Remove unused casting to set --- doc/api.rst | 1 + doc/whats-new.rst | 5 +- xarray/__init__.py | 2 +- xarray/core/computation.py | 110 ++++++++++++++++++++++++++++++- xarray/core/dataarray.py | 26 ++------ xarray/tests/test_computation.py | 96 ++++++++++++++++++++++++++- xarray/tests/test_dataarray.py | 2 - 7 files changed, 216 insertions(+), 26 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index ae4803e5e62..1814b874b3e 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -24,6 +24,7 @@ Top-level functions full_like zeros_like ones_like + dot Dataset ======= diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 963a0454f88..e60d98340c9 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -38,6 +38,10 @@ Documentation Enhancements ~~~~~~~~~~~~ +- Addition of :py:func:`~xarray.dot`, equivalent to ``np.einsum``. + Also, :py:func:`~xarray.DataArray.dot` now supports ``dims`` option, + which specifies the dimensions to sum over. + (:issue:`1951`) - Support for writing xarray datasets to netCDF files (netcdf4 backend only) when using the `dask.distributed `_ scheduler (:issue:`1464`). @@ -49,7 +53,6 @@ Enhancements as orthogonal/vectorized indexing, becomes possible for all the backend arrays. Also, lazy ``transpose`` is now also supported. (:issue:`1897`) By `Keisuke Fujii `_. - - Improve :py:func:`~xarray.DataArray.rolling` logic. :py:func:`~xarray.DataArrayRolling` object now supports :py:func:`~xarray.DataArrayRolling.construct` method that returns a view diff --git a/xarray/__init__.py b/xarray/__init__.py index 3e80acd1572..1a2bf3fe283 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -6,7 +6,7 @@ from .core.alignment import align, broadcast, broadcast_arrays from .core.common import full_like, zeros_like, ones_like from .core.combine import concat, auto_combine -from .core.computation import apply_ufunc, where +from .core.computation import apply_ufunc, where, dot from .core.extensions import (register_dataarray_accessor, register_dataset_accessor) from .core.variable import as_variable, Variable, IndexVariable, Coordinate diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 858936aad6c..685a3c66c54 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -6,13 +6,14 @@ import functools import itertools import operator +from collections import Counter import numpy as np -from . import duck_array_ops, utils +from . import duck_array_ops, utils, dtypes from .alignment import deep_align from .merge import expand_and_merge_variables -from .pycompat import OrderedDict, dask_array_type +from .pycompat import OrderedDict, dask_array_type, basestring from .utils import is_dict_like _DEFAULT_FROZEN_SET = frozenset() @@ -937,6 +938,111 @@ def earth_mover_distance(first_samples, return apply_array_ufunc(func, *args, dask=dask) +def dot(*arrays, **kwargs): + """ dot(*arrays, dims=None) + + Generalized dot product for xarray objects. Like np.einsum, but + provides a simpler interface based on array dimensions. + + Parameters + ---------- + arrays: DataArray (or Variable) objects + Arrays to compute. + dims: str or tuple of strings, optional + Which dimensions to sum over. + If not speciified, then all the common dimensions are summed over. + + Returns + ------- + dot: DataArray + + Examples + -------- + + >>> da_a = xr.DataArray(np.arange(3 * 4).reshape(3, 4), dims=['a', 'b']) + >>> da_b = xr.DataArray(np.arange(3 * 4 * 5).reshape(3, 4, 5), + >>> dims=['a', 'b', 'c']) + >>> da_c = xr.DataArray(np.arange(5 * 6).reshape(5, 6), dims=['c', 'd']) + >>> + >>> xr.dot(da_a, da_b, dims=['a', 'b']).dims + ('c', ) + >>> xr.dot(da_a, da_b, dims=['a']).dims + ('b', 'c') + >>> xr.dot(da_a, da_b, da_c, dims=['b', 'c']).dims + ('a', 'd') + """ + from .dataarray import DataArray + from .variable import Variable + + dims = kwargs.pop('dims', None) + if len(kwargs) > 0: + raise TypeError('Invalid keyward arguments {} are given'.format( + list(kwargs.keys()))) + + if any(not isinstance(arr, (Variable, DataArray)) for arr in arrays): + raise TypeError('Only xr.DataArray and xr.Variable are supported.' + 'Given {}.'.format([type(arr) for arr in arrays])) + + if len(arrays) == 0: + raise TypeError('At least one array should be given.') + + if isinstance(dims, basestring): + dims = (dims, ) + + common_dims = set.intersection(*[set(arr.dims) for arr in arrays]) + all_dims = [] + for arr in arrays: + all_dims += [d for d in arr.dims if d not in all_dims] + + einsum_axes = 'abcdefghijklmnopqrstuvwxyz' + dim_map = {d: einsum_axes[i] for i, d in enumerate(all_dims)} + + if dims is None: + # find dimensions that occur more than one times + dim_counts = Counter() + for arr in arrays: + dim_counts.update(arr.dims) + dims = tuple(d for d, c in dim_counts.items() if c > 1) + + dims = tuple(dims) # make dims a tuple + + # dimensions to be parallelized + broadcast_dims = tuple(d for d in all_dims + if d in common_dims and d not in dims) + input_core_dims = [[d for d in arr.dims if d not in broadcast_dims] + for arr in arrays] + output_core_dims = [tuple(d for d in all_dims if d not in + dims + broadcast_dims)] + + # we use tensordot if possible, because it is more efficient for dask + if len(broadcast_dims) == 0 and len(arrays) == 2: + axes = [[arr.get_axis_num(d) for d in arr.dims if d in dims] + for arr in arrays] + return apply_ufunc(duck_array_ops.tensordot, *arrays, dask='allowed', + input_core_dims=input_core_dims, + output_core_dims=output_core_dims, + kwargs={'axes': axes}) + + # construct einsum subscripts, such as '...abc,...ab->...c' + # Note: input_core_dims are always moved to the last position + subscripts_list = ['...' + ''.join([dim_map[d] for d in ds]) for ds + in input_core_dims] + subscripts = ','.join(subscripts_list) + subscripts += '->...' + ''.join([dim_map[d] for d in output_core_dims[0]]) + + # dtype estimation is necessary for dask='parallelized' + out_dtype = dtypes.result_type(*arrays) + + # subscripts should be passed to np.einsum as arg, not as kwargs. We need + # to construct a partial function for parallelized computation. + func = functools.partial(np.einsum, subscripts) + result = apply_ufunc(func, *arrays, + input_core_dims=input_core_dims, + output_core_dims=output_core_dims, + dask='parallelized', output_dtypes=[out_dtype]) + return result.transpose(*[d for d in all_dims if d in result.dims]) + + def where(cond, x, y): """Return elements from `x` or `y` depending on `cond`. diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 8c0360df8a9..3c022752174 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from . import duck_array_ops, groupby, indexing, ops, resample, rolling, utils +from . import computation, groupby, indexing, ops, resample, rolling, utils from ..plot.plot import _PlotMethods from .accessors import DatetimeAccessor from .alignment import align, reindex_like_indexers @@ -1926,7 +1926,7 @@ def real(self): def imag(self): return self._replace(self.variable.imag) - def dot(self, other): + def dot(self, other, dims=None): """Perform dot product of two DataArrays along their shared dims. Equivalent to taking taking tensordot over all shared dims. @@ -1935,6 +1935,9 @@ def dot(self, other): ---------- other : DataArray The other array with which the dot product is performed. + dims: list of strings, optional + Along which dimensions to be summed over. Default all the common + dimensions are summed over. Returns ------- @@ -1943,6 +1946,7 @@ def dot(self, other): See also -------- + dot numpy.tensordot Examples @@ -1968,23 +1972,7 @@ def dot(self, other): if not isinstance(other, DataArray): raise TypeError('dot only operates on DataArrays.') - # sum over the common dims - dims = set(self.dims) & set(other.dims) - if len(dims) == 0: - raise ValueError('DataArrays have no shared dimensions over which ' - 'to perform dot.') - - self, other = align(self, other, join='inner', copy=False) - - axes = (self.get_axis_num(dims), other.get_axis_num(dims)) - new_data = duck_array_ops.tensordot(self.data, other.data, axes=axes) - - new_coords = self.coords.merge(other.coords) - new_coords = new_coords.drop([d for d in dims if d in new_coords]) - new_dims = ([d for d in self.dims if d not in dims] + - [d for d in other.dims if d not in dims]) - - return type(self)(new_data, new_coords.variables, new_dims) + return computation.dot(self, other, dims=dims) def sortby(self, variables, ascending=True): """ diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index ebd51d04857..88710e55091 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -14,7 +14,7 @@ join_dict_keys, ordered_set_intersection, ordered_set_union, result_name, unified_dim_sizes) -from . import raises_regex, requires_dask +from . import raises_regex, requires_dask, has_dask def assert_identical(a, b): @@ -744,6 +744,100 @@ def test_vectorize_dask(): assert_identical(expected, actual) +@pytest.mark.parametrize('dask', [True, False]) +def test_dot(dask): + if not has_dask: + pytest.skip('test for dask.') + + a = np.arange(30 * 4).reshape(30, 4) + b = np.arange(30 * 4 * 5).reshape(30, 4, 5) + c = np.arange(5 * 60).reshape(5, 60) + da_a = xr.DataArray(a, dims=['a', 'b'], + coords={'a': np.linspace(0, 1, 30)}) + da_b = xr.DataArray(b, dims=['a', 'b', 'c'], + coords={'a': np.linspace(0, 1, 30)}) + da_c = xr.DataArray(c, dims=['c', 'e']) + if dask: + da_a = da_a.chunk({'a': 3}) + da_b = da_b.chunk({'a': 3}) + da_c = da_c.chunk({'c': 3}) + + actual = xr.dot(da_a, da_b, dims=['a', 'b']) + assert actual.dims == ('c', ) + assert (actual.data == np.einsum('ij,ijk->k', a, b)).all() + assert isinstance(actual.variable.data, type(da_a.variable.data)) + + actual = xr.dot(da_a, da_b) + assert actual.dims == ('c', ) + assert (actual.data == np.einsum('ij,ijk->k', a, b)).all() + assert isinstance(actual.variable.data, type(da_a.variable.data)) + + # for only a single array is passed without dims argument, just return + # as is + actual = xr.dot(da_a) + assert da_a.identical(actual) + + # test for variable + actual = xr.dot(da_a.variable, da_b.variable) + assert actual.dims == ('c', ) + assert (actual.data == np.einsum('ij,ijk->k', a, b)).all() + assert isinstance(actual.data, type(da_a.variable.data)) + + if dask: + da_a = da_a.chunk({'a': 3}) + da_b = da_b.chunk({'a': 3}) + actual = xr.dot(da_a, da_b, dims=['b']) + assert actual.dims == ('a', 'c') + assert (actual.data == np.einsum('ij,ijk->ik', a, b)).all() + assert isinstance(actual.variable.data, type(da_a.variable.data)) + + pytest.skip('dot for dask array requires rechunking for core ' + 'dimensions.') + + # following requires rechunking + actual = xr.dot(da_a, da_b, dims=['b']) + assert actual.dims == ('a', 'c') + assert (actual.data == np.einsum('ij,ijk->ik', a, b)).all() + + actual = xr.dot(da_a, da_b, dims='b') + assert actual.dims == ('a', 'c') + assert (actual.data == np.einsum('ij,ijk->ik', a, b)).all() + + actual = xr.dot(da_a, da_b, dims='a') + assert actual.dims == ('b', 'c') + assert (actual.data == np.einsum('ij,ijk->jk', a, b)).all() + + actual = xr.dot(da_a, da_b, dims='c') + assert actual.dims == ('a', 'b') + assert (actual.data == np.einsum('ij,ijk->ij', a, b)).all() + + actual = xr.dot(da_a, da_b, da_c, dims=['a', 'b']) + assert actual.dims == ('c', 'e') + assert (actual.data == np.einsum('ij,ijk,kl->kl ', a, b, c)).all() + + # should work with tuple + actual = xr.dot(da_a, da_b, dims=('c', )) + assert actual.dims == ('a', 'b') + assert (actual.data == np.einsum('ij,ijk->ij', a, b)).all() + + # default dims + actual = xr.dot(da_a, da_b, da_c) + assert actual.dims == ('e', ) + assert (actual.data == np.einsum('ij,ijk,kl->l ', a, b, c)).all() + + # 1 array summation + actual = xr.dot(da_a, dims='a') + assert actual.dims == ('b', ) + assert (actual.data == np.einsum('ij->j ', a)).all() + + with pytest.raises(TypeError): + actual = xr.dot(da_a, dims='a', invalid=None) + with pytest.raises(TypeError): + actual = xr.dot(da_a.to_dataset(name='da'), dims='a') + with pytest.raises(TypeError): + actual = xr.dot(dims='a') + + def test_where(): cond = xr.DataArray([True, False], dims='x') actual = xr.where(cond, 1, 0) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index f42df1cbabb..059e93fc70c 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3200,8 +3200,6 @@ def test_dot(self): da.dot(dm.to_dataset(name='dm')) with pytest.raises(TypeError): da.dot(dm.values) - with raises_regex(ValueError, 'no shared dimensions'): - da.dot(DataArray(1)) def test_binary_op_join_setting(self): dim = 'x' From b430524facb48724ebfb2e3686be3d6f7a80aea1 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 12 Mar 2018 13:31:06 -0700 Subject: [PATCH 055/282] Support __array_ufunc__ for xarray objects. (#1962) * Support __array_ufunc__ for xarray objects. This means NumPy ufuncs are now supported directly on xarray.Dataset objects, and opens the door to supporting computation on new data types, such as sparse arrays or arrays with units. Fixes GH1617 * add TODO note on xarray objects in out argument * Satisfy stickler for __eq__ overload * Move dummy arithmetic implementations to SupportsArithemtic * Try again to disable flake8 warning * Disable py3k tool on stickler-ci * Move arithmetic to its own file. * Remove unused imports * Add note on backwards incompatible changes from apply_ufunc --- .stickler.yml | 1 - asv_bench/benchmarks/rolling.py | 5 +- doc/api.rst | 7 + doc/computation.rst | 18 +-- doc/gallery/control_plot_colorbar.py | 3 +- doc/whats-new.rst | 32 +++- xarray/core/arithmetic.py | 71 ++++++++ xarray/core/common.py | 11 +- xarray/core/dask_array_ops.py | 5 +- xarray/core/dataarray.py | 4 +- xarray/core/dataset.py | 6 +- xarray/core/groupby.py | 3 +- xarray/core/npcompat.py | 2 +- xarray/core/variable.py | 7 +- xarray/tests/test_nputils.py | 4 +- xarray/tests/test_ufuncs.py | 234 ++++++++++++++++++++------- xarray/ufuncs.py | 7 + 17 files changed, 317 insertions(+), 103 deletions(-) create mode 100644 xarray/core/arithmetic.py diff --git a/.stickler.yml b/.stickler.yml index db8f5f254e9..79d8b7fb717 100644 --- a/.stickler.yml +++ b/.stickler.yml @@ -6,7 +6,6 @@ linters: # stickler doesn't support 'exclude' for flake8 properly, so we disable it # below with files.ignore: # https://github.com/markstory/lint-review/issues/184 - py3k: files: ignore: - doc/**/*.py diff --git a/asv_bench/benchmarks/rolling.py b/asv_bench/benchmarks/rolling.py index 79d06019c00..52814ad3481 100644 --- a/asv_bench/benchmarks/rolling.py +++ b/asv_bench/benchmarks/rolling.py @@ -1,9 +1,8 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np import pandas as pd + import xarray as xr from . import parameterized, randn, requires_dask diff --git a/doc/api.rst b/doc/api.rst index 1814b874b3e..5f0d9d39667 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -358,6 +358,13 @@ Reshaping and reorganizing Universal functions =================== +.. warning:: + + With recent versions of numpy, dask and xarray, NumPy ufuncs are now + supported directly on all xarray and dask objects. This obliviates the need + for the ``xarray.ufuncs`` module, which should not be used for new code + unless compatibility with versions of NumPy prior to v1.13 is required. + This functions are copied from NumPy, but extended to work on NumPy arrays, dask arrays and all xarray objects. You can find them in the ``xarray.ufuncs`` module: diff --git a/doc/computation.rst b/doc/computation.rst index bd0343b214d..589df8eac36 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -341,21 +341,15 @@ Datasets support most of the same methods found on data arrays: ds.mean(dim='x') abs(ds) -Unfortunately, we currently do not support NumPy ufuncs for datasets [1]_. -:py:meth:`~xarray.Dataset.apply` works around this -limitation, by applying the given function to each variable in the dataset: +Datasets also support NumPy ufuncs (requires NumPy v1.13 or newer), or +alternatively you can use :py:meth:`~xarray.Dataset.apply` to apply a function +to each variable in a dataset: .. ipython:: python + np.sin(ds) ds.apply(np.sin) -You can also use the wrapped functions in the ``xarray.ufuncs`` module: - -.. ipython:: python - - import xarray.ufuncs as xu - xu.sin(ds) - Datasets also use looping over variables for *broadcasting* in binary arithmetic. You can do arithmetic between any ``DataArray`` and a dataset: @@ -373,10 +367,6 @@ Arithmetic between two datasets matches data variables of the same name: Similarly to index based alignment, the result has the intersection of all matching data variables. -.. [1] This was previously due to a limitation of NumPy, but with NumPy 1.13 - we should be able to support this by leveraging ``__array_ufunc__`` - (:issue:`1617`). - .. _comput.wrapping-custom: Wrapping custom computation diff --git a/doc/gallery/control_plot_colorbar.py b/doc/gallery/control_plot_colorbar.py index a09d825f8f0..5802a57cf31 100644 --- a/doc/gallery/control_plot_colorbar.py +++ b/doc/gallery/control_plot_colorbar.py @@ -7,9 +7,10 @@ Use ``cbar_kwargs`` keyword to specify the number of ticks. The ``spacing`` kwarg can be used to draw proportional ticks. """ -import xarray as xr import matplotlib.pyplot as plt +import xarray as xr + # Load the data air_temp = xr.tutorial.load_dataset('air_temperature') air2d = air_temp.air.isel(time=500) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e60d98340c9..947d22a9152 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -32,27 +32,53 @@ v0.10.2 (unreleased) The minor release includes a number of bug-fixes and backwards compatible enhancements. +Backwards incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The addition of ``__array_ufunc__`` for xarray objects (see below) means that + NumPy `ufunc methods`_ (e.g., ``np.add.reduce``) that previously worked on + ``xarray.DataArray`` objects by converting them into NumPy arrays will now + raise ``NotImplementedError`` instead. In all cases, the work-around is + simple: convert your objects explicitly into NumPy arrays before calling the + ufunc (e.g., with ``.values``). + +.. _ufunc methods: https://docs.scipy.org/doc/numpy/reference/ufuncs.html#methods + Documentation ~~~~~~~~~~~~~ Enhancements ~~~~~~~~~~~~ -- Addition of :py:func:`~xarray.dot`, equivalent to ``np.einsum``. +- Added :py:func:`~xarray.dot`, equivalent to :py:func:`np.einsum`. Also, :py:func:`~xarray.DataArray.dot` now supports ``dims`` option, which specifies the dimensions to sum over. (:issue:`1951`) + By `Keisuke Fujii `_. + - Support for writing xarray datasets to netCDF files (netcdf4 backend only) when using the `dask.distributed `_ scheduler (:issue:`1464`). By `Joe Hamman `_. - -- Fixed to_netcdf when using dask distributed - Support lazy vectorized-indexing. After this change, flexible indexing such as orthogonal/vectorized indexing, becomes possible for all the backend arrays. Also, lazy ``transpose`` is now also supported. (:issue:`1897`) By `Keisuke Fujii `_. + +- Implemented NumPy's ``__array_ufunc__`` protocol for all xarray objects + (:issue:`1617`). This enables using NumPy ufuncs directly on + ``xarray.Dataset`` objects with recent versions of NumPy (v1.13 and newer): + + .. ipython:: python + + ds = xr.Dataset({'a': 1}) + np.sin(ds) + + This obliviates the need for the ``xarray.ufuncs`` module, which will be + deprecated in the future when xarray drops support for older versions of + NumPy. By `Stephan Hoyer `_. + - Improve :py:func:`~xarray.DataArray.rolling` logic. :py:func:`~xarray.DataArrayRolling` object now supports :py:func:`~xarray.DataArrayRolling.construct` method that returns a view diff --git a/xarray/core/arithmetic.py b/xarray/core/arithmetic.py new file mode 100644 index 00000000000..3988d1abe2e --- /dev/null +++ b/xarray/core/arithmetic.py @@ -0,0 +1,71 @@ +"""Base classes implementing arithmetic for xarray objects.""" +from __future__ import absolute_import, division, print_function + +import numbers + +import numpy as np + +from .options import OPTIONS +from .pycompat import bytes_type, dask_array_type, unicode_type +from .utils import not_implemented + + +class SupportsArithmetic(object): + """Base class for xarray types that support arithmetic. + + Used by Dataset, DataArray, Variable and GroupBy. + """ + + # TODO: implement special methods for arithmetic here rather than injecting + # them in xarray/core/ops.py. Ideally, do so by inheriting from + # numpy.lib.mixins.NDArrayOperatorsMixin. + + # TODO: allow extending this with some sort of registration system + _HANDLED_TYPES = (np.ndarray, np.generic, numbers.Number, bytes_type, + unicode_type) + dask_array_type + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + from .computation import apply_ufunc + + # See the docstring example for numpy.lib.mixins.NDArrayOperatorsMixin. + out = kwargs.get('out', ()) + for x in inputs + out: + if not isinstance(x, self._HANDLED_TYPES + (SupportsArithmetic,)): + return NotImplemented + + if ufunc.signature is not None: + raise NotImplementedError( + '{} not supported: xarray objects do not directly implement ' + 'generalized ufuncs. Instead, use xarray.apply_ufunc.' + .format(ufunc)) + + if method != '__call__': + # TODO: support other methods, e.g., reduce and accumulate. + raise NotImplementedError( + '{} method for ufunc {} is not implemented on xarray objects, ' + 'which currently only support the __call__ method.' + .format(method, ufunc)) + + if any(isinstance(o, SupportsArithmetic) for o in out): + # TODO: implement this with logic like _inplace_binary_op. This + # will be necessary to use NDArrayOperatorsMixin. + raise NotImplementedError( + 'xarray objects are not yet supported in the `out` argument ' + 'for ufuncs.') + + join = dataset_join = OPTIONS['arithmetic_join'] + + return apply_ufunc(ufunc, *inputs, + input_core_dims=((),) * ufunc.nin, + output_core_dims=((),) * ufunc.nout, + join=join, + dataset_join=dataset_join, + dataset_fill_value=np.nan, + kwargs=kwargs, + dask='allowed') + + # this has no runtime function - these are listed so IDEs know these + # methods are defined and don't warn on these operations + __lt__ = __le__ = __ge__ = __gt__ = __add__ = __sub__ = __mul__ = \ + __truediv__ = __floordiv__ = __mod__ = __pow__ = __and__ = __xor__ = \ + __or__ = __div__ = __eq__ = __ne__ = not_implemented diff --git a/xarray/core/common.py b/xarray/core/common.py index 85ac0bf9364..337c1c51415 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -6,8 +6,9 @@ import pandas as pd from . import dtypes, formatting, ops +from .arithmetic import SupportsArithmetic from .pycompat import OrderedDict, basestring, dask_array_type, suppress -from .utils import Frozen, SortedKeysDict, not_implemented +from .utils import Frozen, SortedKeysDict class ImplementsArrayReduce(object): @@ -235,7 +236,7 @@ def get_squeeze_dims(xarray_obj, dim, axis=None): return dim -class BaseDataObject(AttrAccessMixin): +class DataWithCoords(SupportsArithmetic, AttrAccessMixin): """Shared base class for Dataset and DataArray.""" def squeeze(self, dim=None, drop=False, axis=None): @@ -749,12 +750,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.close() - # this has no runtime function - these are listed so IDEs know these - # methods are defined and don't warn on these operations - __lt__ = __le__ = __ge__ = __gt__ = __add__ = __sub__ = __mul__ = \ - __truediv__ = __floordiv__ = __mod__ = __pow__ = __and__ = __xor__ = \ - __or__ = __div__ = __eq__ = __ne__ = not_implemented - def full_like(other, fill_value, dtype=None): """Return a new object with the same shape and type as a given object. diff --git a/xarray/core/dask_array_ops.py b/xarray/core/dask_array_ops.py index 5524efb4803..4bd3766ced9 100644 --- a/xarray/core/dask_array_ops.py +++ b/xarray/core/dask_array_ops.py @@ -1,8 +1,7 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np + from . import nputils try: diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 3c022752174..efa8b7d7a5e 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -10,7 +10,7 @@ from ..plot.plot import _PlotMethods from .accessors import DatetimeAccessor from .alignment import align, reindex_like_indexers -from .common import AbstractArray, BaseDataObject +from .common import AbstractArray, DataWithCoords from .coordinates import ( DataArrayCoordinates, Indexes, LevelCoordinatesSource, assert_coordinate_consistent, remap_label_indexers) @@ -117,7 +117,7 @@ def __setitem__(self, key, value): _THIS_ARRAY = utils.ReprObject('') -class DataArray(AbstractArray, BaseDataObject): +class DataArray(AbstractArray, DataWithCoords): """N-dimensional array with labeled coordinates and dimensions. DataArray provides a wrapper around numpy ndarrays that uses labeled diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 2a2c4e382ce..03bc8fd6325 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -17,7 +17,7 @@ rolling, utils) from .. import conventions from .alignment import align -from .common import BaseDataObject, ImplementsDatasetReduce +from .common import DataWithCoords, ImplementsDatasetReduce from .coordinates import ( DatasetCoordinates, Indexes, LevelCoordinatesSource, assert_coordinate_consistent, remap_label_indexers) @@ -298,7 +298,7 @@ def __getitem__(self, key): return self.dataset.sel(**key) -class Dataset(Mapping, ImplementsDatasetReduce, BaseDataObject, +class Dataset(Mapping, ImplementsDatasetReduce, DataWithCoords, formatting.ReprMixin): """A multi-dimensional, in memory, array database. @@ -2362,7 +2362,7 @@ def dropna(self, dim, how='any', thresh=None, subset=None): array = self._variables[k] if dim in array.dims: dims = [d for d in array.dims if d != dim] - count += array.count(dims) + count += np.asarray(array.count(dims)) size += np.prod([self.dims[d] for d in dims]) if thresh is not None: diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index b722a01ec46..7068f8e6cae 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -6,6 +6,7 @@ import pandas as pd from . import dtypes, duck_array_ops, nputils, ops +from .arithmetic import SupportsArithmetic from .combine import concat from .common import ImplementsArrayReduce, ImplementsDatasetReduce from .pycompat import integer_types, range, zip @@ -151,7 +152,7 @@ def _unique_and_monotonic(group): return index.is_unique and index.is_monotonic -class GroupBy(object): +class GroupBy(SupportsArithmetic): """A object that implements the split-apply-combine pattern. Modeled after `pandas.GroupBy`. The `GroupBy` object can be iterated over diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index 8f1f3821f96..af722924aae 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, division, print_function -import numpy as np from distutils.version import LooseVersion +import numpy as np if LooseVersion(np.__version__) >= LooseVersion('1.12'): as_strided = np.lib.stride_tricks.as_strided diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 66a4e781161..5ec85c159a2 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -10,7 +10,8 @@ import xarray as xr # only for Dataset and DataArray -from . import common, dtypes, duck_array_ops, indexing, nputils, ops, utils +from . import ( + arithmetic, common, dtypes, duck_array_ops, indexing, nputils, ops, utils,) from .indexing import ( BasicIndexer, OuterIndexer, PandasIndexAdapter, VectorizedIndexer, as_indexable) @@ -216,8 +217,8 @@ def _as_array_or_item(data): return data -class Variable(common.AbstractArray, utils.NdimSizeLenMixin): - +class Variable(common.AbstractArray, arithmetic.SupportsArithmetic, + utils.NdimSizeLenMixin): """A netcdf-like variable consisting of dimensions, data and attributes which describe a single Array. A single Variable object is not fully described outside the context of its parent Dataset (if you want such a diff --git a/xarray/tests/test_nputils.py b/xarray/tests/test_nputils.py index d3ef02a039c..d3ad87d0d28 100644 --- a/xarray/tests/test_nputils.py +++ b/xarray/tests/test_nputils.py @@ -1,8 +1,8 @@ import numpy as np from numpy.testing import assert_array_equal -from xarray.core.nputils import (NumpyVIndexAdapter, _is_contiguous, - rolling_window) +from xarray.core.nputils import ( + NumpyVIndexAdapter, _is_contiguous, rolling_window) def test_is_contiguous(): diff --git a/xarray/tests/test_ufuncs.py b/xarray/tests/test_ufuncs.py index 64a246953fe..91ec1142950 100644 --- a/xarray/tests/test_ufuncs.py +++ b/xarray/tests/test_ufuncs.py @@ -1,67 +1,185 @@ from __future__ import absolute_import, division, print_function +from distutils.version import LooseVersion import pickle import numpy as np +import pytest import xarray as xr import xarray.ufuncs as xu -from . import TestCase, assert_array_equal, assert_identical, raises_regex - - -class TestOps(TestCase): - def assert_identical(self, a, b): - assert type(a) is type(b) or (float(a) == float(b)) # noqa - try: - assert a.identical(b), (a, b) - except AttributeError: - assert_array_equal(a, b) - - def test_unary(self): - args = [0, - np.zeros(2), - xr.Variable(['x'], [0, 0]), - xr.DataArray([0, 0], dims='x'), - xr.Dataset({'y': ('x', [0, 0])})] - for a in args: - self.assert_identical(a + 1, xu.cos(a)) - - def test_binary(self): - args = [0, - np.zeros(2), - xr.Variable(['x'], [0, 0]), - xr.DataArray([0, 0], dims='x'), - xr.Dataset({'y': ('x', [0, 0])})] - for n, t1 in enumerate(args): - for t2 in args[n:]: - self.assert_identical(t2 + 1, xu.maximum(t1, t2 + 1)) - self.assert_identical(t2 + 1, xu.maximum(t2, t1 + 1)) - self.assert_identical(t2 + 1, xu.maximum(t1 + 1, t2)) - self.assert_identical(t2 + 1, xu.maximum(t2 + 1, t1)) - - def test_groupby(self): - ds = xr.Dataset({'a': ('x', [0, 0, 0])}, {'c': ('x', [0, 0, 1])}) - ds_grouped = ds.groupby('c') - group_mean = ds_grouped.mean('x') - arr_grouped = ds['a'].groupby('c') - - assert_identical(ds, xu.maximum(ds_grouped, group_mean)) - assert_identical(ds, xu.maximum(group_mean, ds_grouped)) - - assert_identical(ds, xu.maximum(arr_grouped, group_mean)) - assert_identical(ds, xu.maximum(group_mean, arr_grouped)) - - assert_identical(ds, xu.maximum(ds_grouped, group_mean['a'])) - assert_identical(ds, xu.maximum(group_mean['a'], ds_grouped)) - - assert_identical(ds.a, xu.maximum(arr_grouped, group_mean.a)) - assert_identical(ds.a, xu.maximum(group_mean.a, arr_grouped)) - - with raises_regex(TypeError, 'only support binary ops'): - xu.maximum(ds.a.variable, ds_grouped) - - def test_pickle(self): - a = 1.0 - cos_pickled = pickle.loads(pickle.dumps(xu.cos)) - self.assert_identical(cos_pickled(a), xu.cos(a)) +from . import ( + assert_array_equal, assert_identical as assert_identical_, mock, + raises_regex, +) + + +requires_numpy113 = pytest.mark.skipif(LooseVersion(np.__version__) < '1.13', + reason='numpy 1.13 or newer required') + + +def assert_identical(a, b): + assert type(a) is type(b) or (float(a) == float(b)) # noqa + if isinstance(a, (xr.DataArray, xr.Dataset, xr.Variable)): + assert_identical_(a, b) + else: + assert_array_equal(a, b) + + +@requires_numpy113 +def test_unary(): + args = [0, + np.zeros(2), + xr.Variable(['x'], [0, 0]), + xr.DataArray([0, 0], dims='x'), + xr.Dataset({'y': ('x', [0, 0])})] + for a in args: + assert_identical(a + 1, np.cos(a)) + + +@requires_numpy113 +def test_binary(): + args = [0, + np.zeros(2), + xr.Variable(['x'], [0, 0]), + xr.DataArray([0, 0], dims='x'), + xr.Dataset({'y': ('x', [0, 0])})] + for n, t1 in enumerate(args): + for t2 in args[n:]: + assert_identical(t2 + 1, np.maximum(t1, t2 + 1)) + assert_identical(t2 + 1, np.maximum(t2, t1 + 1)) + assert_identical(t2 + 1, np.maximum(t1 + 1, t2)) + assert_identical(t2 + 1, np.maximum(t2 + 1, t1)) + + +@requires_numpy113 +def test_binary_out(): + args = [1, + np.ones(2), + xr.Variable(['x'], [1, 1]), + xr.DataArray([1, 1], dims='x'), + xr.Dataset({'y': ('x', [1, 1])})] + for arg in args: + actual_mantissa, actual_exponent = np.frexp(arg) + assert_identical(actual_mantissa, 0.5 * arg) + assert_identical(actual_exponent, arg) + + +@requires_numpy113 +def test_groupby(): + ds = xr.Dataset({'a': ('x', [0, 0, 0])}, {'c': ('x', [0, 0, 1])}) + ds_grouped = ds.groupby('c') + group_mean = ds_grouped.mean('x') + arr_grouped = ds['a'].groupby('c') + + assert_identical(ds, np.maximum(ds_grouped, group_mean)) + assert_identical(ds, np.maximum(group_mean, ds_grouped)) + + assert_identical(ds, np.maximum(arr_grouped, group_mean)) + assert_identical(ds, np.maximum(group_mean, arr_grouped)) + + assert_identical(ds, np.maximum(ds_grouped, group_mean['a'])) + assert_identical(ds, np.maximum(group_mean['a'], ds_grouped)) + + assert_identical(ds.a, np.maximum(arr_grouped, group_mean.a)) + assert_identical(ds.a, np.maximum(group_mean.a, arr_grouped)) + + with raises_regex(ValueError, 'mismatched lengths for dimension'): + np.maximum(ds.a.variable, ds_grouped) + + +@requires_numpy113 +def test_alignment(): + ds1 = xr.Dataset({'a': ('x', [1, 2])}, {'x': [0, 1]}) + ds2 = xr.Dataset({'a': ('x', [2, 3]), 'b': 4}, {'x': [1, 2]}) + + actual = np.add(ds1, ds2) + expected = xr.Dataset({'a': ('x', [4])}, {'x': [1]}) + assert_identical_(actual, expected) + + with xr.set_options(arithmetic_join='outer'): + actual = np.add(ds1, ds2) + expected = xr.Dataset({'a': ('x', [np.nan, 4, np.nan]), 'b': np.nan}, + coords={'x': [0, 1, 2]}) + assert_identical_(actual, expected) + + +@requires_numpy113 +def test_kwargs(): + x = xr.DataArray(0) + result = np.add(x, 1, dtype=np.float64) + assert result.dtype == np.float64 + + +@requires_numpy113 +def test_xarray_defers_to_unrecognized_type(): + + class Other(object): + def __array_ufunc__(self, *args, **kwargs): + return 'other' + + xarray_obj = xr.DataArray([1, 2, 3]) + other = Other() + assert np.maximum(xarray_obj, other) == 'other' + assert np.sin(xarray_obj, out=other) == 'other' + + +@requires_numpy113 +def test_xarray_handles_dask(): + da = pytest.importorskip('dask.array') + x = xr.DataArray(np.ones((2, 2)), dims=['x', 'y']) + y = da.ones((2, 2), chunks=(2, 2)) + result = np.add(x, y) + assert result.chunks == ((2,), (2,)) + assert isinstance(result, xr.DataArray) + + +@requires_numpy113 +def test_dask_defers_to_xarray(): + da = pytest.importorskip('dask.array') + x = xr.DataArray(np.ones((2, 2)), dims=['x', 'y']) + y = da.ones((2, 2), chunks=(2, 2)) + result = np.add(y, x) + assert result.chunks == ((2,), (2,)) + assert isinstance(result, xr.DataArray) + + +@requires_numpy113 +def test_gufunc_methods(): + xarray_obj = xr.DataArray([1, 2, 3]) + with raises_regex(NotImplementedError, 'reduce method'): + np.add.reduce(xarray_obj, 1) + + +@requires_numpy113 +def test_out(): + xarray_obj = xr.DataArray([1, 2, 3]) + + # xarray out arguments should raise + with raises_regex(NotImplementedError, '`out` argument'): + np.add(xarray_obj, 1, out=xarray_obj) + + # but non-xarray should be OK + other = np.zeros((3,)) + np.add(other, xarray_obj, out=other) + assert_identical(other, np.array([1, 2, 3])) + + +@requires_numpy113 +def test_gufuncs(): + xarray_obj = xr.DataArray([1, 2, 3]) + fake_gufunc = mock.Mock(signature='(n)->()', autospec=np.sin) + with raises_regex(NotImplementedError, 'generalized ufuncs'): + xarray_obj.__array_ufunc__(fake_gufunc, '__call__', xarray_obj) + + +def test_xarray_ufuncs_deprecation(): + with pytest.warns(PendingDeprecationWarning, match='xarray.ufuncs'): + xu.cos(xr.DataArray([0, 1])) + + +def test_xarray_ufuncs_pickle(): + a = 1.0 + cos_pickled = pickle.loads(pickle.dumps(xu.cos)) + assert_identical(cos_pickled(a), xu.cos(a)) diff --git a/xarray/ufuncs.py b/xarray/ufuncs.py index f7f17aedc2b..d9fcc1eac5d 100644 --- a/xarray/ufuncs.py +++ b/xarray/ufuncs.py @@ -15,6 +15,8 @@ """ from __future__ import absolute_import, division, print_function +import warnings as _warnings + import numpy as _np from .core.dataarray import DataArray as _DataArray @@ -42,6 +44,11 @@ def __init__(self, name): self._name = name def __call__(self, *args, **kwargs): + _warnings.warn( + 'xarray.ufuncs will be deprecated when xarray no longer supports ' + 'versions of numpy older than v1.13. Instead, use numpy ufuncs ' + 'directly.', PendingDeprecationWarning, stacklevel=2) + new_args = args f = _dask_or_eager_func(self._name, n_array_args=len(args)) if len(args) > 2 or len(args) == 0: From e792681d52ee8d5f1b855713363f9def6e0a319c Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 13 Mar 2018 09:03:54 -0700 Subject: [PATCH 056/282] Docs and other minor updates for v0.10.2. (#1984) * Docs and other minor udpates for v0.10.2. * Update build deps * Misc doc fixes * Removal failure on warning in doc build --- .travis.yml | 4 ++-- README.rst | 2 +- doc/_static/ci.png | Bin 0 -> 200731 bytes doc/api-hidden.rst | 7 +++++++ doc/api.rst | 12 ++++++++++++ doc/contributing.rst | 2 +- doc/whats-new.rst | 10 ++++------ xarray/core/arithmetic.py | 12 +++++++++--- xarray/core/coordinates.py | 4 ++-- xarray/core/dataarray.py | 1 + xarray/core/dataset.py | 12 ++++++++---- xarray/core/indexing.py | 8 ++++---- xarray/core/rolling.py | 7 ++----- 13 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 doc/_static/ci.png diff --git a/.travis.yml b/.travis.yml index ee8ffcc4d5e..cee21bd87c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,8 +102,8 @@ script: - flake8 -j auto xarray - python -OO -c "import xarray" - if [[ "$CONDA_ENV" == "docs" ]]; then - conda install -c conda-forge sphinx_rtd_theme; - sphinx-build -n -b html -d _build/doctrees doc _build/html; + conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc; + sphinx-build -n -j auto -b html -d _build/doctrees doc _build/html; else py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing --verbose $EXTRA_FLAGS; fi diff --git a/README.rst b/README.rst index 40491680c3f..94beea1dba4 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ xarray: N-D labeled arrays and datasets .. image:: https://zenodo.org/badge/13221727.svg :target: https://zenodo.org/badge/latestdoi/13221727 .. image:: http://img.shields.io/badge/benchmarked%20by-asv-green.svg?style=flat - :target: https://tomaugspurger.github.io/asv-collection/xarray/ + :target: http://pandas.pydata.org/speed/xarray/ **xarray** (formerly **xray**) is an open source project and Python package that aims to bring the labeled data power of pandas_ to the physical sciences, by providing diff --git a/doc/_static/ci.png b/doc/_static/ci.png new file mode 100644 index 0000000000000000000000000000000000000000..f535b594454574c6a439ba6c7fdd0f4939d489fb GIT binary patch literal 200731 zcmeFYg?A;rjyRZfIGuEuJIu@s9cE^x8)jx^W@hX#Gcz+ohq=Ri!`XcA&Cbm3p7VSE zz~19q<+3HYEZLGRtHR}F#o%GFVL?DZ;3dR`6+u8C5kWw}^r0cYN~WTSd_X|pD9we0 zTU{Qjs@^|3S zJSjM2hTRq^P-UKI2o*Ayl(!U@^=9}i*Lyhd{gV@Jo* zVKoLI>t)tmeV9IWnXO03Q!^O{%iAb|f086f)VDEz&N#f;j*X=t3^RmAlO=@=Q5{v5 zYAr89Lo5*Jp_Yh<2rY+r2@R^PQ$Ll-EG6#M*Dc41M>8jI%~iAa~Kf#RthKJU~=pC2qoY$>&LP} z*9)O}FptvKceeJ`@wVC3l^sQbN!UntQn723C*t50j)S&<#rwerfP5b{kOET@AVX|& zT5%cBE6+jfg;0zG1@ATBXXYT1SR)RZ;fO={2qFQg&$ss<*99F7e4!W8Tv`AvqV~K? zBChXM8s#MTW;C$va7b|@-za30PYIR$D}DFs}-e64f~3QT?|a8(Mw)!OnPM z_30Q{I}DZc_wuSkTBWuU;z_A(r|`b6g+_jn-h~plp4gd z7v(|JBf$(8HjFnSK^x;A4EvcLO(Pr;k8A{14__wiBNjtc5>Hly))v`FtTT#pfD)0< zq720OduWEG>3=C=IlFLTd{4F>(k<-C=vQVSJ&Eer z%frBi9$l>eHw`s263w7k9YQsFX~6e|YdORjfNxJU-K)_Grd?y;_HBBgWY_V=_zBXB zT{kdysAm7d4dEl1pBxkU97Y%XhXD^^AVapiI0flB$xp(pz-?LbUEFwaBjl>^%j-W< zWG2X_NU2EdNI(=hB&~4VL0#nVBDs3<=ePn1ZAH4WxW=5uRHp=|=tYSaepTYkxa={) z1JwF7ZGJk^E5cj?J>p-)Km``c6jXXimSDMb2@QcYfwB_qd?&@1QcY#cas<|#3<({6 z+Co*O83la#?eem+^fLGIXeE@gDK%;(?mQQTDxC{rnM@NgzyjI)LE~YgPNPy|M`Mc< z*Ji|}oLv6#eCh)AU(=J=6Z}Q2Q{pAgQ?Cd2)5BBN)0IU%b9ojT)^99imIsze7C+}( z3+AdM<{IaXEFD;nEmAD;mO87}Du!kq<|CHJ=QMveiB(9PwXsS3k>pm-tmzW>?0Cew z_yIZ0U|NgCGb}j#W0+I4ltuy$QyA_lZw7(ZC9c4g?++zI7+~OX|9@Wo$ zP#b+yhH3rwK9fL4$jt%g5Kn|-czSFfyC+_u#o1|wWxJGJmjROjU1C6?JaPD(Mx`yiWO0^dQrV_`s&S}sW^$dT zud>&wXHCelye-td{TcgC^bYSkN@N{+18q6YrG_F6FHLaD84bO9cT0VW-;{akcmGic?cDMJ+k{Kvg$noL^pdQZnvLraUXyDB#+q%tEv^9O zE~g+*f#IMvREChBP>xWt(5v{-=x96yv87lywlO1>@nX3GhN!o*lk=p@#;)78#2?Cy zvfi?-ztIiZ%rk8p&mrG8Ne7K{q_QW@<{k?n3fW`3_s8vU&)Qd;<{lz5J2Drm)~hBq z`+1kV3tz1faAPrJjqxyeVRHO)?s#x`etERHA9~>4zfAn;P|(1Hb_;QGc%) z?{;6T?#%CupMSsDJXqf`KkUD#Jn=rV1CL*??tRhqapJIQ*>_a(0C_Ce;ya$r7@j{~ zl@ZZVv6WF5kt=bt5IYH<6v8ybMa5ObfHH`1!sf(;Vonuav6+e=JkCC=VK!!X41Q|oJo@6GLoB#FX=c_ zRXdNr?dREkKU}YNI(Mde`F>#gr1px*6S<{aCh8`9@C=XxoJCV-<XX^VctaQDoy)pg$r{Tsf3Fg2S&m@~(-U~@E|K65K{>zf%ovo~dtWzFT; zcmBu;>exIba`b!3I?I&LcCK^!IseX;K96ox^IFUwARy7V{XG$VnVw0L#aCg3KEFp9 ztsF!0yDj~rZl^I!w{T;TaoNQDgH#XtO(Y!}H+m|%ukNsx%U`t;y$PQSy`1h&B3-}7_DTFVM6V=GO`A}LE%E$TfE z@5k`{@az*{6F^&*jaH}qSD81N6}DH~>Mg^@Vz-Hss)8!l=3twOvf6TrP1%i{miNCg zgSZ$zcTZIgC4JL-*?KPFu1yy)o8HayUG)yJyE$koX33r9mREyaYCwpKiVBy~t>)ae zo+l6vf?wBrz{PD_+$fG!x6yt12ZM&lP-5 zw%gTe$c2#G{q_;Hth^j`r-DzTkFhPK1f|UqW?5GjS@TX`lecsn(~)dzP7M3Yz2~?) zQ#x>M4)?iZL!m^qg*hp_R15kJooaIzYrfmiGkh`44NZ&cR=u*0Q}?EBXX&$WOP>{4 zJ%o0j56I);!%PKsEnCu!c1O>>i)QXO?wKo-ZEtV-E8Xq)X9Q@32s{fu8gHYg(mlQA z&-Lx2WtUFg(ynLwjr~{L&X`m=BRM`-IuA)-&6mjojYG%dV|X1x9W^~^x4CD<+xnW0 zYq_dl?A`V5&QJ88JsF^QwdwAO5jY@A*Tozm4RWA%AZ*mWZpD5w_TaHVsM5#@c^pKI_Q-~D4EQZ| zX#AJgZER5fa;Tli#-1KqCErgBx0t@mZ*~?Lr?RDpL%2>Ag#p#+KfnDN?*WZ0wXYv^ zq}ck}JHXh9YdC^{pp*Xd2bEAHxds6N%`sP2cT$&;<^tGS(-|1q8XD8NS=)UUm#AFyhQ3U@&rP*4#otmbS!iXM0~IW z1Oz+|MkZW}!lM6${`JI5Wai{#$3;)?>gr18%1meLU`o%($;nC2z(mi)MEeCn>*#Ld zWZ*_?<4F8(BL6oXVPi*tgSnlPxvdSsKXeTYZJnKXiHQDb=)ZseeotdJ^Z#kd#__+p z_0>W8e`@F%=@{t$oAwtf&p)MH^5$;FR_em$*2XrDUv2O)va&Mq{0rg#Q}sVh{x4LG z|3T$oWBK2h|EuP|FnQ?z>B0Z%(ZAXCuhK7m@xk)Y|2OaXVA-HKHNM30(_C0i`Ro4e zAK83KuJ-Ff@$dUr9yIb?3oXeS1cV<%LRdiA4fH%4Mjw4?3HWIqtw5tiijHjsp@tqD zCaf2S@f}r9fL1w09Z^2h%eX?lS8x`x5*!r~^_zEMyPrIfejmh}yfTFpnlitJFu1~K zD64`v10~_S*YUK6X^wjWIW@@})D-i~q|WsFCimTBmg5zt0%^TJ4on=#e@JHF1!nxS zW_GFI;(q_f)Jwn*_6_RWe;^mf4-JZqC>VGBpW2NBfrtJHM#BH?|C|WH&G?}O`u{T^ z3xOTnf9z}!Vjd_VI1Ivn1f2EjW&BUCMuI5l2N{G&!t);izZf_2ANleBGmih8a~!_{ zQpCAQHb!6?{<_|i#Q$S-^!+myj`|D#-y=re6gnx3$M4|GHyU0wvOrUAP>dQGNy$`m zG~|B^S{dqVWR?d~O+=;f3v8)T&_JHDT17+_5KoiAMB4KJah`FaMP=wxdN#!K48iMN zdt#>W4~eP)^iJJ-dl6`g5N%XGw8{d9O;ut2wsDE%YM-%mqJ=NN)VfX$<^0pH#%;~rNOT00?yOWSpgB+v-u9UN*f z5s`Jk`r!@v)s+(=!CzYre){;->@Z7Xy0_KOFfk&)4Ciw|__OhvYw zSZhW9xjmbI-=)=2o?hGL(+AaLf2lJ0>6&`yMayG1MwT`^BBFaWf|R;tuXf|XwyS~O z#zJG8)79n@b$9f`>W=*+6yDB#g6xhl zr(qN>t0MeuBs-EEe^yxWBUJnQ{-Ne*3i5;}t#USEQ*n`DWzMZ9&9nAF7XfIxZyN@N z2+0e(o6qWi+koVs<6R9*JBTYBq4Ru6x1nW_=;vxX3TMO+@YXkw zV`GtRnm}pgBg-7D-!-?(?_i`EdYR(>W1q#{$5wp>F?B9S=@#zJ?*Uo={NNB-OsaAv6k7zwBDL*wrD?DSGG;xAUZOos{axjaeBx@h@ zA2GYudC*9mr@Z!gHwZ&^6Ru+Q=bgLi`lrzzEIS_*s4{W~syk$TIuTdkz2*s!(H^$d z@8-0%eSX~4hCu%Mx1AXOG6#ZKes1eh`k&)4;@+GyJgcW{Zx^wy9wHBqoSMLSlzzBiu9pg zTz;N5vX-x@TX|M+!8!L0PP>B%N`R&mM=|FZ6!@&eV427B4S?VAP{&3v`Mw`KitAn` z$|k*_=x(S}0xrhnQuo^V7P12hH7A~Q2=`kKNYD}5kYd+?&Q)~AEK*4-?9DeUROP(& z&9pqOSFRL&q2EJ@?5ml!B9TmbY0fh@HVwvckqJQmNzh_n7R%1ajmOVV4Yb6>N-pEq zrb;X10mXwnDXJm_Q(!n`kmBE4BVq7}I%i|Z<gAQdA4jS^%KekADb*r;B2LyNJ&*wFyT=ZG3$HI94kgvPkp z^RK$JjuyrIbCl*1WleKGLAw?c`VadWh(gVnxzi%OmT|u9=p;)|&IVqUeMp(~(2pwY zyc18DO%H{ido?Vkt{BGngOBGZVPzPBZ&l{v8D*&0%c>=C(1yxb^rC2%E|r^uy)#=2 z*EetrLrOc(u(&k+IAvW77Hu83`42l|;$02NldWo?&S!-WO|)}6+S&e^_b>^2P+T_N zc~uEzh?fmqN=VH;4KSP~$#D+6Y^{*IDB5cDOSD|@&=BgRkl+trv#t&k(M03}PHEFy zN#*cspUiLa_PPEHA7-(H@x6pR{EWzc1%0K+L3B64iw18ABet6u$Ye}1svY{um44}Z zyQLMO3(Ob}-mn4G{VgL5W4$ylVDKe`2wzwhM=7+MXoWM{P~*M4DUUE{N>NG(gxK`d zZ=O_AfP;b=P>wWcTAt{a_}41Ug7?doS$JbCYlArzcj;KZzA9bIr%GMa9KKct2}-6O zTIEP+^ASjUvmsHS6EDeICQCJSbd(J?*|9NJU};d<@RwpeQ|V27B*%_Q(ifjt21um) zie_fmRj71IsRkAA$1P$5E8^4UTB=$Ie-`TEy`ksy{tXmwZ?36`5<%YDW-hl<6NXaT zEB`~@akxwh7Qi5ZL`I#p+%(Xcx-fS){xZ7V11ZsyaWWYr($s2mThne9Aad0coLTN@ zoV~EDS-X$lfJ?T&kMW`^JE!qI!<$UF2z2)+1Hp-Zdvr3$)mV98xt^2@*|4onX#>eI zerwyA8Guz=S=o~2?O1WcXe7S$JG>FjzjaZ0^w(l?DWv%lt&{Z~nARZU9vLfo4>3f+ zN7;ci4Kn><#a$mOG1z@#roDKrsJ*nOD4e?$d;=`O@!1||XW+^Xea}4~W(;ri@^J!V zd6AIPUall`I3p#j9!c|8i@f`>zc7PPnAeJl!HS+JAZy|8SfAsxGE>hFNa{sB3A&ICC15-6IU&*`90-ScC=b$pix1Z=P z67ketE<(2b=zo|k>9~1t{SExQ^Ib$u)v;&APR=i_RL7}^Yk}n<2lV}D*RgZLc>I)l z5zj!H7aQ8v|1{OxTXnVAVCC12Y!0);oaL{WJn@*vfK-IJPE`xr$L9@AGJVAAO1JwI zCuy<`>Ho|MT@BglRsikVtRqUaq)j=5g}qdmj<|iiY^LD5LF0SBC4f~BqcfrpKpysLu`91*H3;-oV#035uO;b#q-6+Mi=9i2n6vkdSau?bX!un zK_}-;J^%Bw>TJRlor6~T(1R+aIVka=_5L{D*Bfwu{QN+&Ia4$v7(n6)5B=R3La=eM z>+{YVvO=l&p~-twB14pm{xTC}` z1HgRs!v4;f7{+F3kVB>|`EX@Gfpp@c%jMLZ}Cj(R!zC3-n&NOMM9BM030uj zxlBi}Cw1GDzVR=QJTNH%2Z9@D@EIAJu$nuhW~y zWcNM<7XMTbKl>9;t2q6wObOVw-H40#llX8mUA|dSkG`_;9gwTv7m)mPjKD-m+y4An_fOB)Hdt+dG z-$5n#(w$uCbi#`YL^zE(${Bwfcj9+7NUEt*BR#~QKx+ngTgmNgs?{@iPV-o~o`M^4 z-zR@0-5j0Cq51^)v%@(>gTFK1{6%$zJn#69%xF}7FLD*A4%V20#keXOYqQU5j$&9+Qg@uof^ItxuU+` z1+(``zSsAPY`-w$4XF9K@7QSaO+K=YR31$Ig51c+3d-d1Zw|Z?$Q|Q7 zxtYHdKt6fcq}8f(xzJku=+kXeGY1B$R@o_q@?p|ApVcwkGh$u(*r{Jaeed5uP}Vw@ z(jo++%#f`R`z+%ULx$?cdbGa|e4Ym6d@kUoieYvP z0QlXWDB;?PuuomjXX9<7wzVI$91dBn1AXZ)wy)DhbFyRc<4q&9@6!9-Vk~@E_X2XV zue964+QV9E>h;cde2zTw%0PJi9<4Llvh1MBt#`f71?z zxjYzmNrAC^wzYjD{M;c&9@W8PGP1nBv$S&u%*bfcwOd)K^`#m|nr|RiHN8qrf5_Pt z@gGL_9adEffN@u7YkBgq-l6J{0Ld(iGpU5CT>c~y@9Q)})+y#5YqI^=AFZxh?G_s6 zGZ?CpgtuKb#S&`7P+|Fv88PBJhkpPm_+>Bzi~qE z@h=L;bzM?%1fQJG2nl-RX6GCIk3GM)<%%hjl+PjZ)mjxF?jXGnHIY{B0OeG?o?`IO z?W}-Hgi4n2Ivsw2r=N&?jK|?o+fucbyV#a$m5LGCp2F1ycL#ZunwGQLm>$*ce_wUD zdEX3AzxT)z$;tbfbotW$i1_T_F8WbM*X)gOZ$&-B6aSl8lh`*xcp9My|JPW`N9_ya zCH4l1gWEw9r`APyGUMvTFG!W`%NOwHexZkksg3H0+<$hmc67*jpu5`V4-t(Tnc3w5 zowDtQV8bO>i3tVqJQwC0^EA@=dlRqWjd15r%f26qQAvkyXQrJb^d)^?y3xgy(0Le= zc(k6}H4>a>@r;KjUbh<)Iz}tb1_{OvV$h=I^AoXv8S z`Qgn#jvnONYl4(1nUV4?$=%~WgDT@fl*Du0;cE65qeRD0PN=z;mI&Nfyq7{n=cJCu z`VYDaMla2xxSsmsSC{oE$1k!ex2&FjzNb;0D_>iDrp#X9k!)^JE*^Urud5q(1~1?b z0=uG$TWwnM@515LO(--rRfStlBA&~_Il38f_jA_{b7~*g-%jzQPya}8Us4rXUwZi2 z(B?OtIa4a3PsI1n|GCR2#TH{ZJU8xX|K25MBht=BcCLN#O1hu_$9jdyUH!HLM!gEo z-7FkyCN@+JqAbQY5Dw;Nr|tz2Uiu7q=wp@r^b4`Ws{hn$6n>!8miG@VoV$C}VV{}T z5qO=r^=%%ffU2QtCpdf4kL zshb`cWSg4<3tR7d*c2*lkhkD^F2=DP53+Cdc3?TNrBfz%k+C*?C`!7$Hih^sBn7=c zpI5s^C2|c~slrG&nu&9Dqq?r*HGZh>WHd(JdRcg1a@u&?!!^EOqRD0lFC-w!&YfiQ zKn8L_v<{TsdTN*vY!Ez01NeN8{$R)r_OjG4EoHXADCYXkDXd<(yM#UcT|wL1TFl>W z-+8#WvunG%_5MaZ3=|l*P*XUWLHXm0t(!YPp>T3+J2ePDaMU&?xUn8Ij5V9Q!;u|{ zh60(QNk02M@`DkceT>_~xTQcR(yr=BA;#=G^Fa3-WDpV0A6n1_RtO)v_Y0EhULx9R zem5||xjxntmsepi35%o^3)HQT%+~GN*Sl5Os0$wO8b~!0ybE5W%o6hUo|z9c==bAr z(ELcH{*c8>@%qj0thl6%LdJ&@QCY)y+P zuH5KHnye;f1vYrP&IX)JQDTn-((a-XzYF8B9eZzFv9O?pP|#*5LIrX#pFV_KS8)({0I z%uU5{GstguYx$U(r|+IF#deEe!|HYoAeC2w=mpV!3QG zO#za%mVY^l%C@ilOpot|9=7gM&bn$`dxAI}S9aw`uj(;}4mEt{@eed~Jvm-wwN=S} zb*r3OgdN1=)^U03jB@iQ@l42ilVGJnoXp3LIy!3>%`9br3h)j@#hIo(Bk>KGF$B0y zyldw%TBrCy`DlcdphNJ;Id`BWhHMMET=&MOgifXPSpOL z+|JponG)I#WxBBN&YfRA&=|#B*nG049gElB&>7J*Kgi+Y0DDzDf6NG7l8mk}E)Ho6 zm+rSX-0?fsqK5dF!(vDK%U{voHanp*JG}pAo{?zNtMq4FJBYVWJp{36r#?mgz7HvA z-0QleOjUg+Jmw9*YF>e+l=1nC8o|N&XfK#!^J^l6k1MLSzv*URwH6{!-5prau@M*y zw1hoDL5B8d#DZR@qkai6)5y;6fh*G(T$pPJeqC0`pou9^@Jf)wjW@tlKd%?!^ZglH zk|XV6G8UW##9jT=#8y;m9foz*>=ODrFKB}QX+3FeGi_m3sO9$8Q+F$4ul3%lIH${a z5+H8lTbC@FG=eha*ZGJ^^TG37sKg3dDdbFXSS3tLg1(g8KgF2rrG;*4;w4&m{la=6 zsTg-TsbPJtcT;rpvhR)X2$P58J>Br(*l$teq%fM1_-;<^Iy1Zmrz3T&@`j~H3)>go z;1C*%)XWixDc!!rMr7uVLRMRM<}zu@A7{^}JM7*}969M`i5`7kSU^(C&Io55VQ@mU z-=Ui7z4erotiY6@^O9ffmUT8w3`n~@;0ihB`T@P_oulidh%ckYaUAR#InrM#)hND6Q}d;95+#+w1;VN`KzZkH`417NE57o#H{n zf%2EjiZ3um$LDJSgcJ)J+t258Uew)*5}08noI73E3G0f*ooRfISwWtZ=Og4t>osfo zNj@Y#yUBScYQY^YpzmuP0A*SPn5Eyd)87vc051r^bMu5|{00phUyJ!4; z-)fL#S-$#YProN36|VMigSU=f9AS}l9-v}aWNEF3U~CF4kxzXpDEopC)_>kdio@91 z#a5_}YCl10c29n^@l~yq!BQ@AWtnV)!VO%Rm6obG`EhnpslZh{-LUSTh+?dw&vuFyiWBR zMsXw@o@xBhBV#5!eI?T7_C`}d8)DjI%`s8$&%dlp5Wm;nQ3uofs_kG8{{pnpQ zgVi_}Fz8%Fk6@-PgOTvGM2FO^?p+!*D3OvxkAE^i?vT^U0c&PS;a0nA(5HGC9$J@D zD90dU!toD^F>i~8angb9NYQrc1D`{8XRP$XmRLU{ymyWTomVbmcd_*$BSGtm?CjHx zF8SpHag%2?x2-O3Z#{YJar@no!VUJYL}!R1e7|B!F`i`&h`eix0~DwN|d+3k(sZ7bXCgVO~meZ{F~ysox{|K|YD?Do!K zHbJAKppqRT=(6$;AWKL_E57;w|DSJ~}J;c~&UkR6hX08ZnYRmpj z;Aqt4PU6&ak0fEKyA*ZtL!G zy_A!u;~GGDtKOK4Ce#MeMaPPn9-&L5RD2orj|@|-``-S=jwqK278`Pc8dEsQ@%@h6 zfiNtsXdd3769#E4T;*ATzc|F(s4xeX+9*r_nc;a_rj$ih0_OQ+=_IVb!!1(rkJ7Mo z%(s|O0iQt>rpAw(X^bZex#@w$jTur1Dr0%0x2BeYOO{Bt7o++F_QeWZaWa!zbH;0QQGO`0C}_i+X+ULf=e|zD z(@04B$1s8@jy+}Q!nXch0;jm|pS^>&;qqNw6Y+C3uJXRBQ`=yH{g9m$^u1+rPjd7U zkHm8L;enwv+4oOkd=yM zVW555{zd0`27*j580iV)e!MaqYl(u)!*!!VgPq>Jor z?FwwYOUI~1xcjz&VQJke}HJ()}(DKaDP>@!S? z`hpK^(=RW9HQB#x#=$iyJOCwNfm zVXQ>$r2DfH-;y+|o@(Upsz7JmA<9hv(Rc&E(|FP(BN@xLST7gv^u7zbxvC>96VIC= zh~oa+rGVH09t$lBcY$zvi`J#Uy7UQm0(pV&6`k2Tiv3?AB2?T>|2073re_?*HCd=VeRf z#i0?P!PvfIrRh7xy}KJ3q&6YKQvI{B#a)GPR9i#EuIS&6nH56oajE4ydsImAZa&rS zgshdrXX2-}7fbi*9KbI%vFz9bEV=ZVB|=E>E9S{#t(*apGb~X`T5?mKQ=WmeD%ck- zqAOzJEMPo^JAS%Fk0Tm>GBCd6>$&v%etdVY5B?2vv}kTTz{t4qS=M0Okk3<;B(gQw z9r$T8xUy|C$U&@MM``)C6W;nO-`_L$+26+Nj_Q8ur!GW;X_Uu5hA{Sa6M(read-6m za^c?Up2RwQn1RDW3x@D$7MgbDSo^q!8a1(M9#&M{)>a_TetBGAr8nCZi50YZ8YATq zJG8O8sqo7bifprDYB<%)m9Gw)*|P%wo6e5~ie2_C_jN>n(0u35@U+<;-rHtVgL@XG zHEWot-TH{X0+wirq-0Kjg6&y#2Vs*#;7?0(kwy5#I$Hz$%2ZFeddDksdPWpF-OK$CF(&`yxv<*O7-iQXKgb~l0kL*P$tzU4i2>OtNUyFuGv}&4ERA@{hpxQ z5JrNhdFRg64s4ooT*c#Pkrm6#JhSZ|(!L}RINeN^j8QfC;WrPVd}^{puam4q+cU{t z^NPiYCY(q+e%W{*Liqw)EJrCbea1UGkd_ObT%YGG47JY0xAG?@RmSBW?J&%;spo=d>)x)AR2#MBmt*v8B`vm$!}*5};=*5_&Iq=`&C)j%13?Pc!PyDsV$1cflx{kc+*5 zX17y%;!Ugs7p!hMos?cKn~qK*I%c_Oi{P$_<^w571yc&D|E=e`yxprG5*p?#l9$kLEUIx-Joa5_{$!& z_EM=&OsU?l$qisem8H z3asDx4jbE`GqPPGL(K5y>mXcoY>-j&EJ{?-J+l!!m}|HjXxvwJR)5>lN_E_HH(H#q zq!5k&o1!pCC4JhUR_4Zm7r%sy&|^aed!R)TvQn3_yW|<$AoTpx$>;j|-k1)}oBV;I?5W_lgr;ljUO-F6@yEvb~2Da;^r4Xu$Z)1;!qRy4UrQ zZF(qPDZ)CsaE(*NDYwXI!zm7Tw z!;qRkOSpe?Gz=IIzq~R5Hd{ka03)g|REq15-x=|WJfoq;`J3{zTxHA;m=VQYK4CGu zO}eRgkFym{cpM5<*f$j8z`Yf}!b3ag zQr>Q(t7)Y~3{4V7Jb{($!+p-9;at)3RD|y-sgAQEQdio#s3Uxykui=Q5|lOJWO)We zJ-^t@*iSQ5+~=dzjaqv#zwR&qaIuq{bW{yX{N^x#TsU2kBiHSV9Z{G*i&(V z;v11HKi@p7ef=Yt<=eN^T3pC8Vxcv5? zpXvgpEr@htB(V{L&S(vxCr@no=u5}%n(0MG1QuvHjf(l%?VZ`=dh{l`4Er|4S@tf% zHo$~0$Er6@vF~c)+5xpZS2VhR2ODN}Fv=p7qe|6&O{Q)KDTSHaGrzp|XV0oC5>R7s zX;NbdZ^6*7+w8nOP-x88n$%xjN+jc%bLr2V(sX)C*Jp7BPuOCGRfpsYCKGD2VAv@b zCXFSi;NINQ$S08BzNb=e(@yI+&!9Ehz0Se5oG1guqk<7{sz?miv^!|xt;825^ z^|Pyf^x|BG0TAH7#B>3;ZxKe9z6&7RDDZtjc}jz3Z6q# zeN~g?Kb^36v}PIRd0;`FJoc1(eBb-}RK(#y;mcF$BT|7{)kHcCi&(CvUds(8!Q4DS zs78n-fs>D=*lCUW9BsV1*YSbn>5{BfsM~DgW~T(Qn^fe1?W(5~kYZ@Zk@K3|sUE)d z2llCa0;w8*!UnK({YVk>Zj&NxqO(k3(t$BndD&vD9jt(N_FmZf_b@nyamBV1*d9>SMsMNk3yXJJ zU~-aLXYL?8@%6b{yp9m_33ZU>CfRTm_o3uc(luVUjZ12YCwy7MB?mgG=ScOQ*aMsP z8p%$9YXnsX9s6i|3!NC_RlEWwpx_d)29U2Vbwd zD5oYSYyEv#Hg831%W|HM_*&`gm0x3mi$0jk!P3jwJ+NAeG?E*#^Bs+%#8F=DTPBn< zX?vpdADC!k0_Oc8KY1gF^~i9};3j0d)ts{5NtKd+&lw5S#~+C^w1|~Ze7JI1Ggs~iyEVmhCzD}Jj!0*Qh*nrSsTpX z=AQF*n#TzzJzPC~EVmG1+WE9O3L0VjGKe3V8k(L!Z;rDt5cWPsPl z9U4qyXYY?kJW*){P56u!?6SQhNo%~|6falp@+T7^F_HI{R`!fD)*@`uG+)6DTzAwe zO~JJ#wECCy>i4CHT<}4u+2cI^IuFgyFU@E>Czo{E;+gzL)m3ql3L%wZNmMM|sDtUQ zM)qV0{i)OT>@L3XW)#Xw08V*de5BODK*@<4T6BWGm*G(EAVV4AXH2tXFX=3^Qr4UzJChqOo;&+Ux=l{V?;l{1_aBe_f4Wth`% zf#IK|QYoW`9Gy3n-SQ40e|P|sg&fTeZvRqm(z_}@)wW^5uV|c9T$3ifayhjF()5;) z>tu{XGD+)S3F~Q5*o_jWUORW!O%hw1&Ch?!1C}y}RW(!R>8Lw=rYMbFIWRK2$Cx~u6wlRJh(f%h@9me z^P0Y8ez37nYNBBC~CdUSW&4G1yYE`#UNOZ9uasG;*b^v~8hM$+%CPi$KeC1)hq0HoAJ zGj`c#KSEOEU)&1vv2ylP4mq1tGm8FemB-r36R!~gMPK`X#n`PkBHi_+ONT41e z5)0a==mv>}*hvld?XCIvU!j++Hr70#A@6l?B$%}ncD7XO5z~9HBMNCibiG&Rb}juk zZ7bbkZ^@rQu|q2!<_&XF6PLf#kc=yn6u!I@b05jAk7|kUO`-3X?S535bAdAx5kGRz zLJ?E>3@jbwI4ZPB@cMdj8huhse?#T^AiFkV0Qx;G8SFYOg>SKgHtMNbj%^(w;Gmfv zWx!N9B#M5_L}ZAMyz+Mv4zqhc#8`VR)Re$XH1Bn*ns$KZ(||?Jb4pcyW1Rle@Hr6A z5)&C9my2ZCx;UF0SfH&*WaM;`M?SlS?zWGD5k}H4Gnsa$xQL3kdXO9O4b?!ETHXnExzPHD&9VW5CE_z zOuuv_#3D#>7uWJfMEi~VqYxS%T5lran$bd}Rr0g3QXSe2e=eFp7RRYjTBH6ciq7B< zIXMsDypoL|tg>D+uci`x6=gM#EbM9%Bk&bCgbVYVaE9TBnZ0yD2C2b;rCPbZPVK0X{TnHLj(8^)tMi~l>1%OG?NJ`IOsOW zDUU4x~o+VSl-4h8t`_^-vj8)wBjnV)Tii0JI9%_8Bog z4IW5v&XNvu$8_=Qr^XIOr@D-Aq{f5+HHw~uuPFXrPT{X0O2^U$>Um{vNNbH}ltHu^ z%n7tu?n0Kur6?uYJV8mvKqOJl(%TN$e*MF0rO;-Sq74UGsPVn5rBlb`V5BAmN7f?h zL&;>3a_0#bW9S|=xsB&ZSohtVB8(O}m%Md?!w&MQ$D!oHB%exg9O_;zHSIGRYx@ao zl)g#ULwinm=<=;|T#Ynl(?Iga0NU7F)0`|W1+O~K#_daU?;1Ef7e#hmKW-G=>_1f1 z^+%W!Cl^b28N2lQO(8nEqa|cwt!sN8zm&s41%l-5zoOTs`-oji3f9v zPae~?de~#R(KpUiz7AqBYo(lT&4<6a7AI-rAe-h#uS6!KtL$U3m|Wbx=L$TmnNvB+ ztQ&*-q>x%_92d)r*-ZKEjN_lIxx!VLCkH&FYk%98{)lt8Ut)Kz7JrVpGNN_l;?p)#HAy00&+HuG@}aaj1Lx1#Zk zL%0FFUb<>TFq-dm;8n}fR39#F;R9E!xWT|}S-~iJli=;Dr0s`zYRjjB3D@E6BUntt zaqY4_cDKJag*4{xaPWn~+gt#CyL~J0hZr^U-z8zzOFybI+4w)yJnOt`{xWM*R=@Ry z_xN~bJXy9}LKVb$+KqWW$;3}4HSMF0*6pV60Xq}Df6ZDl0(r-EIF&O%`$h@*HcQ@Y zN^vp?8^skwC3}x!iI@es#%~nS=8-4#BLnHYpNgmsIkI|e2<)CzEs*h>+3 z_cJ$;h#Wb<1ZyYq2)@FJ{VPo{%T$>`4q;q=j=_S5DhNY@ZyqYtf)1K}0QX+*TYjG{ ze8p>H{*Iw3eoV!vHH~sd9Z_v>Fo6i5oEa1P>@9i~-CwdBw=EX#=7s;0BTwhjxni%J zrt!R$48Etgef{a%Jt87M+jKmIVFMAcfus0A*XO6>DqtctC^GlcUcsnMjRg)Pr30@x z3J4C7k}T#SZzhNEqRLw4NLYbY38!Z}Jvg;|?jwiwL7!v4V`>JK4m6G6+5Ck8H#FPE z{SFc21;jzT&zrYU8P0RZxhc`QWhj%et(0$z2Amm@swc+PyZtNjbz0;b3qe+7mu?JO zNB1Z3Xi?iIMI1<@geb_m$6kn;Y90>mOd_cMwtlIv?FWH%1BJPS1ew<9>{_-1>r zgy(>KYQ#WPYY*nl`k|5ki@kRauB_|!g*)unwr$(C?R1ik*|BZgcE@JNw%xI9f4iUe z-l}uXciukt|9h)erE2daYpyxQn8V|@UZ1Xy-gduV(_z{~43bWykFIadHu2CTV0@_{ z0r%b7P%&@S;gq@@a%irFF)3W@ZUbIc#Q_|}=MA17W1>{IC#Er~sVa3Zv^~835BX-} zp$9Klvo*s$*hIIkH^Icd34itdpDDjqF{x^rrdo=}xCnj#z0A9Sm_Fk1;|8oI9v}@T z&OcS$N1VE7aNC}>jjyW;rAN@@SUF$#cRI+sdRBNYyw3AkdZXn$LFM2r2W{Y=_p*g~ z#r|Ap)zmgXFp*WXE>X^*gSXY)#|3dtljD$F;U;X8Q+wd&kIc7u{H4#8)7Q=H9YQx5@AIuM6iuiaKvANI2fOO#Sp?+CU+#8RM z#OH;7L@zQn9N@_-8Sqit_ zCV(ufha?-vh*&15qDqL&OI>z8qGVR@G^ST^)4vM>?dr15S5Bg?a4e1I{$L9kMNkrK zeVw`Fqn^Fq#I4S@rbdv$_F$ukK~lk`2`DSn=h+kEdGXWzVkg-gHtbG>I03txMe(F3 zC-@j3^(4GNCs3+^@#EI+p~uX?j8?nEmd+}Fns#>BXkE_FcY7|muVTrU$hv-*K<4E zC3m_GcY~0Y-z0Z)9jKd>*25d4uNzN;6$V;MV3|(&E1?Sn`hu$8N;h{VWq`pe@LjeK z0IHzLRgK+EGz1U5my^_O-C+7a5%tt4vtQY5L6T$&duQ=-(%Js|bS~rx`lUd%-~km( zQ7HUr@9>=w&#ovKCrPbKw`3eYbYiCUN zo%J#{cTipfXwiRt;k#tk2El%^V;#)CW~|VxYAEn|NNCKnO$yh*3U^yY(OAE&Mfyz! zm3CR(uv_O@jrU|!Jkfi>>lu8>JY9d@#;XE>D4WG5nmud*uY-#DA48w>Tor?!;kzcz zYAaAFE4A7o!W6wg&u4N0aZ$uXaWO$S~VxtI+SMvKBvP_*_?@gsT6 z*ABLcKGtApwUWap>*Ln>%0!*QhkJv0aFf1xd)J9q!l)Gg1+va!yX(Pl90aMqu~aWM z$4sAh|5?jspkB3wKZn(Wf0{X*-&#W9x0Ots@3J&Yt8S4e#oYXovyp-_qrrMECId4QWU3U2@MQ&)Ojp;HPgU>t~`xRw){M;zGIWf9o*RhQ%$#IwE50=9I%s(ZqOqFq(j zdvI|Xo;e=o_-&W7?)oCi#m>2f0V^U#A4{oCaiU;YWr9ASjK7m&{*GXT;$M|4_g3qE zCv-WROc{5{)n4eZ%L?lGnm2eLF{M%6KEIi^4uxOMn*O4=q7MR&fX-nuTFr$h%hJx1Dcjv;4diuZV5%F z9Bej)_M)SEPmMqV(Xe^Ja7k=K1JYuTBjR?_p9C?alKmqd_Ayt_G~4v0%QaF3PzFev zW{wV6zdn-E&DmK_QP(fpRqXOeoNY7W{oPg<^(*^>x_5iRN8D)EX^2-N!(LOvZHx7) z8dSeF(^S@&*v_x|3$L&oSqdWKgfTrxJ_VTh(;s>RKgb!ps@j^b2znLIIlZ%1#vzCdNZS`w(+I6VtoRvE0oJBCX z%=Tcc8ci`yp{P6c8jixp=S5=ZpLd9>cbnle{C=stn~nN@v36!8pff3qQV*{i+34zt z4}&+pSG*0ifuS7p{Y|UAKFh-lZlC>C+g`ig0WR5=Hxd{!JTuvy9;+4g>jmeJQO&I8 zER91v@^_BDBXJk-A4yFBXThZAkE`%?w3^Z>U@&mbJI0^eGwr420x-w8sp&BL=s2s# zGU$@QEoP)aUVU9^Al`;C)Q|tuO0aHuJWD<*ge%$Z(>3yVhqdk8hgKFi1fQ)Av~kNk zN4XTkzn}i{P2&Bg+>?*fJ;p~yjouX`*ESi0^q5L4%<6>-Y^g||MQqX&E9-utYoN}F zZ$yXdl9gc3uyH(DJ4ZcI%%~=nr7%RX>h`c7JQBltlFVuC>|rI%$#9SHDD+q7P@$26 z2P)XYdi3Q)FG!57omxT5%^GE|&qcDpjdGO0jnS8p#3!nkn|!Htm)<1`GnPJu+{)Eu z2p)P-4r7JxL7p46iOz8=aWItpR_vMQZgDry-G>CtQ}_?Ap2&K8tw``8huZfn*3>L6 zc+LWJ!`mz8BqoBrOAQjicr&+Y`KME?^(9HuqV9ySHL=OtY~&Ql$HvDt{Ok7S8RjS5 z?NUeYW^n=P!Vgy1cN;77$;p)Oqn!F){;<^bu|z!~#87YL>ROB5U?@{!afHlosgh#n$^|=xU{lC!w&<c zQuYScvI$oYYXzACN!YV(#3QL?-*YY_M163BGPKg!JBS#E_i)w;{j%m5J$^)k;(Cuk zaN!_r2_{b5>UksaRwrUWv@a5v{}OT84?+o{Nig`|SOC^fZ%e33O+2U{cg?LZCe7s~ z!0#Hh*WTU+2>V!3eh3WNc%Cn!cc%#CVxW-mv=wn!p^Ow_K5(~`cSMW?u4)9K`KOcS zK11W&6I(AG=hCF#ON#oLUN{A2D53N>V8hHfD8M%V>9ahq*x;jnY}8>y`Cu4i2u<{S z#S7kKT*R~;4S7vR75JDcwlarvbtqxa zSy#it{se8`y-pQ1>M0;ou={xMEe}b!E8D+ArjIJIY*2! zN_!vnO$qBH{M6fk-F08FURWa%UNIxMTKDOPoxKTpF^cajmtWNaj83N6p8djIw657( z>r8cW%s-Bnz&bIsVC~p@B~Eq9+OHe-Gb9-dn6ie63Mrh;$R8R4U|P(%OU^M63}Ojc;C$UtaeQ8c|Cl z!(jW+b2YbM40SL2ALh7&wX;5w=mjMnbnEWhpv6cMU-p~q3?5ELt6zx>s)s%kxF(*V$uxGTZXjanTJu0H>gvae( z2)+V3-g|8y4R6QjjaJxLGqjr?T()hEJ;T*+BOj}a2~@~;xE_8D`|%=RXT`iq7V11G*t9T`@h4Wi-Qu!!%0T!QVnzd#kvvfp2cTr-M0NP^>Dl zt#KQ0boDOarJh;icXkJKq2=xB^rdrMw7uIaC3PVahy!}ra=Tl;aCBc$Y`R@@R<0Ui z!6yn*o@2TU>OP1;QoTh&QPE^XF716qno+C@%A{vE9$8;%Jk`}l{oS^xv=Yg-wFzl| zpHI5QR;ia5#LmZlNoXSj-aIu~3M9C9-=Ntw)EFcFnj<3Zd+X)e*TtvW_YE4W{vd6- zdu6B3tz*BhRq?gTwMO5JrF;vewvI}|8nu&5*m5?SqM)kMZp8wpJOqgNGU>z8Uo3@X zKvc@SI4a7{8nSCpL{F2EZ9$8%cu=h9(j+(%RUuTWi(P5WaNeQd%Vkb+{+Ex@uEBBT zhqL3+N`csB8JZ(!W>cU2pLAClP%hW+G_FOa!vtr9Th8&8fN}dxqBubVxtiOdNZy|<$>qtPaAIG zQ~Rl)WbB+9B0ZF%xLT9y9@TCqUD#u!a?L_H1IyjO;zD|!&_e7NF*F`@FVfQcy>e&^ z>){>Sda{g72(NCTca6B~i=^|&N&$f?osx|{{gV7Q_bX9F5=|&MkOc9#vgpK~d3Nd* zG2AU56dUUfWVZ7IzBv)5R4PKy%y=LdWw7Pki5O3DF^UybGW z&!<^068CP|)(g(yTDO+d_S>u$q>Mh!?$?oF+;BzPHYamY@l7@LBer)<*RBX1;HPxR zuT|&=h{1eaU zYJ4buz>;QU93EczFtR^776U45)k5*dsGYn?yRXCSX!Jps46Y4vLKoRHnb+}1tNfoB z?nfA)gXc#Xdqo}X3lwn94R%P;x%WAtn<#R zgW1$Vn@`R7Rm3`DM$H5?*z7B3eDX}%Z2WZeY+{$*Elos!&GucC`SNa9tNt>7j2$F? z8M+q%vZg-XAbHrN9odarA8F_6;UOP=3PU>V#f#Y1AzT3i^RkfI$(?(t<@3pm1{$ms zI4!l2m`4si^Qg}ApVAYWHGY5wl)G+y*8`ZM2*OV)xZ!}SO(YR7++n*e)^CJ{d^I}@ zFyQUF8$3+XE72hEzFL!9IX(<}`8{&f?=koQCRNyX^=`5|{sG5W(uz{8wVc&9IBK3a z?u+Z1@aQPHyJc9ITE(DEP#*UBd2(sO#JhTU+U?}RBv;W7J8`ykHvoL;(dMYNWll?t zpr+~o=C7!Cp$7AINpd;XpchxW1s3s5toj`5m&Xfec9I6v+&hy?YGjPQotgBBTX>f% zG>GkM0JE?X|KV2k4JyRP$~t08SFH)vDm^2iH|xp=){Qy#k9o*$?cCV@E|99YU;NJN zs1Q`rA8?RL@7OL66-t8>9*(f?g98|(c$)0+8{W24boZ(|&#O{u1m7^jHGXiINf);; z&W_!aICseaW+3xOM4k#R_NXj}whkbw&>0C}BJq!+wU5Plf4MAUjqz@F)@5OW_=9fg zvTMHlFtMxW8KtKVqqw`%oFDbQ+Ifx0{D7k>B@s{j)D=$p6f{qY*LIoODU3_&oO(D# za^Mezx(lNNgwCBI%3_q*y;iQ)$U_TxJOI#wG~$x@ zO(j|tpwPkq9u<2~3PLJgv@X73_A6IJ@rEzHsyileVm@xb%n6celkdRAdY3Bt9A)&W z4t)d{nV6|uEegnx!d-Ta64a;D6$L~{!n90Xf91m933VPdWKTl_p(Y|ND03CX2+VfmsJflkCGI#Mwjnl zJVO8U$Or%FkuR>WP*YF`BKNLHP!=^TE4~TZ3Xka^vI){-H?3)I^?)JUaBoa|x>MJ9 z_vA|Es4to5vp?rH93O74;kKs9{?w^_bjNE??>HY${m2{_nboJupZQP^!*F{YNF}CI zoR4|ToiCYb4|?}EG=qB|_trL*ppVC#o#>~XWvSLCLAmUmFLWyCAkc*3LmqfFmb#{^ z3jap=n3R1IUh82Eo~=P8jp?<}bnHwn2G<4~<$c7fSWc5&7_@w?1#Visd`QiWW!-lF ztH#@9)XG{un5i4(&}PW-xOX9a<^0>zbrFX1r`OR6S!s1eOBCvkE)9J!limZ`Zk@g) zPgs{V?#zPR^&!3C5sES*uSBFqXA+|H+C%q4Xv>%Q6(PK44V$#87%z6^>ls=JA1$?b z4&&*Hx5>?Gq-^~Lxc2iTGPT7cQ(DIFcKJ1TZEOzG59eVKIc1Bkc(Gi#xQ_ZgnPyF{ z{t&ysX^IcOFz&4Oe^3^t6i(^DV-2~l{_>jeW%#|VqT@y&J=kb_R@ZDA?afaZq*?r69!_78T- zW2bSyXJGkloOnk}GSUyP&wq``g+@a6wunwz<*_Yc8FKETHu3rb0|HCEd|udyMv5^~ zlr}Dz6EDTQuHqI%LQ^lx0DifP~%tOg!)A~h+!W;#$-c$qZ(A2Mpe%{zCU zYCB(pR$N()u?Df2GdfGgZJDHP?U2MGrTX_jRAB^1h!(lhg; zSHhX>AG!lD0>s{(wL8NnZL;(V{Snv%}5Y2OMQjK3;!B2Dx-G($ggi|BahDyMeSerL!v*QauhdSz1Db1 zvsHef7}U&D$VjZxUB4>DcdbP^rW@af@I4jJs>b1FnR_4WQg3CPFS;-MPv>KzRT;hQ zKyLC1hR09rfL`RB-m2#K9Py^A9_FzngZ6RI&4Qm2Os0s{@tXbW7U*RZcTSJw;Bycc zf}$ateTlUrgvJEr!bsV87y&cznqp){pz}I7t+L2P8)KA5`=Hv%%<)a1$U9vWzr7JS zc2{U=zD+?qwaEf^mDgUM%h9+}L0blmWXC2R_*Dgv8~2uWA#MrFjnuC)xT!Jgo=5h$ zSk5sX?gwO>tskLTUr(pTeJ|iaH(x_^i}sPo-YxK6Ma4AlI&>x1#bcwL`1RJf7%p+@ zTA&8$dejxIaWB!f zvyZ^%8<+w$kZ&9b#F?uG%t9P)>xEsNH~i>{1M_iUhhwjcwda0(T4yDk0#*FkW^l61pR517Ka1$?c3&nk1LK7TaiNW#DH|iAO7QV^?%CDE z25RuxJA?k14GmhEdnc``cno){v9wt`)j5kMILvt``_P4Qk_yt8VHhfZUItQ*&IalH z=iV@v&W2;9n-(VNN2@~O=Y!_`F)JuViW2iuI5rTp~#X5s`UJ7u|X;k$f^}OCRx}N<}y$G@1OAU)S zQF5NZhazMW#GPb7?BkTu$$H|exYjhNe3?C!R(0R-!f_^tEOoa#o3?f92aHTm-> z#!D*(<*Uc2RwaxsJJTa=J=;p-C2l*@=x_p(3yL^nrPRY?;)b=6sF)tLuw%&u0)uLK zSlTl)rQ|$BU2x$hqDq_HQTBMaw8nLYpvFu(+7`cE?G5pu!n;!}`DCz7NS7eOz#MhH zwVD)S7;%GR9*!cYv{Y2EXbs}|UwAFp%Ss9K?0jN0meWCnxxhfl$gwa)4_RG?{e(uk>5eELXR2$Y(VzvZ}Tr^K&unJ&l?y|W2iYTY9`&6uFro+h2W46aXcfskS}ztK>P@xVcndHqu>M##sF2M}Lr23l+7G|a-Ug0G z&NJm7$ETjM5FMUb9ewB@K8$FXO%%qPs9ahFcKe~bdcKC7B9PdD*ZqpPh|ZRV`jv94 zjkO|`8}U84)YHgryn5*iEYz){3VrXEdRUsH$maR{qQJl-W#jg?F@LG0@&H6Ug`jCA znmovWD-rn6MRHFqi%BQBc58PjhEhqS8Rfl)#-go3YeWo7K)>rKJqqXitwUu0Ps;a_ zWlB}B6nvNWYFOd`uF@SX0PLCtYCDos`7Yc)va9t+c2}zke*Z~&|3;!nPKAY+@f=W4 zD_F4BcpBnv)O@vRQ+nh94?I@;-K*r7*++g`v*&_b^HM1ETld&N=owIe%kd|zGZM45 zf%$q%^BMn3Q6J(9Nl(^FiDBRfu8mZ#* zU}YYV?J95zdzeDKN*fA~G%$$*F=)7*K}bKkMq1eMm`gell4{zS!%xCZa`-~`~>Rws06MU4NmJ!ahy^T959Yord z{p>3=*Rm~f0ObG&ObW(5-Mp(1LrohKk+V7z87{3U2;x6Efeq2NlRux7Nd@7wb&AYu zEY*0PiKYIdhu*SyYlt{_up*8)nM@sNO zL`lp+DsXaFn`bGGAtj#x)w)rGw8l|_nrr?ptc2poFpgxKNPoXLYPL~mLe?W?Cj}oK|)WZ8mV+#}IVQLV}Dh6caYQ1#=6V-2X0DbF2d|Mb+ z?^~aq)vL}ke&6Kecq~5QBvBGTZuBpgUpg6pr?SXqUVnJ{mrDT}V%k>$ znk$UxPb1i0E&2%3YH~;_%B?ye{tu(`I%b`;F!W(zb5`) z7X}D6@E^_LFP9(sx1IbseRF@7Hs7g?`77+NFDPJG9e_0_NUkXIKW)bR&(bXRi~p3| z|FV3e@1K}0H-Uoaf7%SopQY0|PKp1*zCVd?EdfB{TWeYpCGLOPOgdm`R@uyZfDZU8 z2WkJ{ThbnpgiHKYBm{t2|K!}$$tGX^8?OD+vGl*b|9|56S3>?jas0nY97~ZvdeiCE zb~}AKLx{I<2OmG5yNcHoDo(jCzaHj2$IuN#@wXy&dT+i~T8(9Vx`SNed*H8pP7WR; zJzQ$8NTP)Q`*Z#~llZEy^cKOz5Xa`bCy4iRLbmGWhZH74*T>_M#HTAB+`f!jEzGO zELv^UH@59wo&`_ki@UjIBBG&GZ$S}hS*_z7pCohSSN@7e<~s{#ZmlKTs3UxlC5`vW z(0{)PfaW0J@!R=CF`f$J)o{FHo!R$>G!aR4m6dve+2>I<1o3y!zT5ezbWj`Gi$lP0 zHQB**F)Lh0d*^*6#kAw^YpKA?-zJEUHpcp!*5&{KO(ZltD3Qky3)COlP&K)Tq_PtD zeG+hj4QAFy1?q9%b4iEq=M1~1s&A~x^mmn61bH^T7S@fEs$W!r0Ju%d=eY+aL+h+A zmX0q}<(o|Np}+j%^o-(~I2KSsjtzfI1-wNlX9eM3bh}?T632axva7++_e^xusMhOS zDy>Lz{UBHf;iZjT%256C1A)iCTwZC6JY0c!w{?tSN{Z(oXOiIxc z3hxs_@y!1D(ybzS&qAOAIap2~2p(8&Hj4B&bDUcRKy1sadax2`2LO`$_6q;S)B%3w z_XoDTUx^)l)>nc3HR&hOzke(Ozmq>^=Z6BX^cVmRmF65HT=Z{R(~J;cA18a|%IM+% zww$hx=C3IJCwPFrKGDkwwnl`7Fr2vp{VO|tV*r+tkKo5MHK6L!>axQ2S9<=5S5#=H z>f2ZV4|H;oXZJUn*r_1`Anz(qqEAqR!2?hB$7cUZZaPE`0M73ci%n2Q#t*>|X|CK& zxBE!%x0`NDWD1 zG+g~^PVv@NGw;-RPXcgH8V@Ph|3;vB9zQNy0foVUFkGdp`WOC~P#aw~B7j>xgJ3}P zH(C(@5h@(QLIhM{MP~E2FC3rgjOosx17K0C`O*9QwXzJ_S(Jv@a5qB7H!v-l<*Wfd zRoTTiw8yTITN> z&;QW<=MV+`-{_vn3H)OffR!>{;uRYV;5%*79cz>}CQ!<`2tX$oUoM8pXAPx&pT2_s zj8Q!IKmB}^QJLcUn}GR(jtcsMgTkW0LSRK&EEfUgVdd6I{EdaBWPhH4p|+1iXYL6L zMAuCh1JzqUbo+Dy^LC-XYcOITDiBM266$ZO<}MK~YN8i7I(%js_p>Mj z`{hpHi@43rL=CTJAtp8~;&CT{2QEc-g)fNf3tyTx?>?Bi%xeXsWd4nNlc@g?37Dcn zVDzEbhyWlBaBpIuCOoOjgaeQ*m9yeV%sq6?O`h45Aiz;y8%+Wgo{yMN@%qPPo6 z&`X^faS=~1f?nP;uM0~opo`WA)%OIGQUz)@Ab0PkhOHRw+lpY)>n# z0C=wf7QDsE@7$KZiQ{i#yGaZLm3Csq+zJjzst-Zkbs}>g)rO40fb!g7roWxD-{Szy zPw4BT3zJ)!u89m71DbKo^>?`B|Dc4+;s`vuth>JN?jF3(H>P;%`Z7=w5rv5M-uGR{ zX;bt&${Ys3&w$+VMhO`^(GRA^*4+>5Mr6Qo`>Pi)`R8>iz%>;DW1~rNN-#@BKidE> z7V*~v;)BCY)y+oYlg?+{UDZl`dnaP*1zuJ>NOY0{3DCyWs%Xp!e+h8}Q~@fO5N70$ z@&p`ZIL9Vuyec^WLY@)Na#}mc<+tvDb54Ap zm272kmDp@J+FVce$?*k!QY0>#6q@EJVwe`ra>0EHkwp7V>m*7iv57bzvlC>GhVis@ zoVyW?)j^-Hj=Ec?9$r+U2d8%p-385;;gV5Vk((M_ZQ$4$u zTdE&X8f0V*MkFi(i?Z0>`+`NrFF7En-K9#@o@ z^_SsBkb+QR)Vy#tTs=QqpU{yfQ1m3I+y~PZ<2i#(gr|kC(GK+45yBHMc6Y(GKj056`u4D^_y@V zZU4fhUu&RrhkbeqxZd7fn>J}{w@85fheJBZ4vK4J)E1{U&hH2=ea%x22$`vI7G{Bj zQO1M141RBYt-XE%-+PNw&UcQ`3IhY3Dx6%NGB~I@=CI&CX*J;Nwnjk+M?@sHGmhfg z=Rvy2{&#W$2YhHfJlZBO_?!9!#D_m{-Jbcayp_=D2E&xL?F)A2#nU$mE%wsmVgpsr z%oy9Pv0ugO)DWfG(6x-P+lRjlq`4zCvFUp+P9qM3x1tbSJgb4Mp1;Bu+}_;|7uSk4 zLH5U&VC#VtRJjX(1Ih!n%oYy&k@DVw!f*PivA4P*)L!6mw2L=p`*hFm`gv%AH>NWo zZ3_uu1d}uYE$E8(_Bu3zs>d_Es2>VJzx66rbSE-AIqI*QK>@75U)XdTO%D+LQg_g^ zR?gwcgpY0^0|?+{$chs$2p;4AcrN^)q-4$;1`?dG~w*-S_3m!2~ha!!fZU7 zf3@4vewNBooC+~A_~vW!PW5~>>ej)3!Vo_N!q;eNwAMiOjZiVh>+(HNkDIP?kRVI}Qy7HuT(Qb>jp#+PP9QTD?sHxHlF7$< zc1BRcoi2w13v`K9dhHba4`E@%m3**XY;aUAGH$m=7N2kaDs(W^UHZ`yY-mz4Pw|i{ zq`=NJr!!EJulJl^;k6^Jg|&|0pThr(1o$}++gXeOIl1bi0`b81eqFehb3ktB(u7w| z=sqO4$hZ$tiUI>gcPAmJl;eo(*C8a(iMcFcb#^Df(QLzxvFGW>cA zPkLN;#f7l>bu}&_Cm^cZ<`*LpR6k)aX4qq$9pNT4{8wB2O*Dl5C#rx@1w>>r^dINc zLPjQQ*?{-2*^}@k!gwa{R-*?cjzGFLsD|!aUQc^}=S=5klXTw#w(QoAv_nH6A@dvo zEk^?4=JPB^9sJ0h;&hmYs&p+5`|z#)uR4GV|Ea{5*05&lrd-sBtvjNsDe|Wh)OZ$)JfQreFSlTB)>&jx8C?}(8=s+?mYsZ)Ef|_#( zk^O?D%bo`V4Lkf$tGvfmLtvUhGqWPn*MQN*TV)%f#+QIL3<06v&}2F#Jn2Zu6|bYs zI8%~m8VlL0_{}?-MovPlL(oul>>NDOG3Zq!c!x@ouD@UzErZ!eBv zTN^ho%2rIH6zyH#MzxDh;Gt)$mY*2^5C;xkknjVd0QVe-&JEp;4_y-g_4NakH5YWs zf!WD=iZ?3r)~J%y<8n6h`r*y|?DLqXZc;B_WX{OD3&IjDbQ&D(HsRTwGBECZk1|O9 z{Feae{0(c(19Te@?%JjWmhB6?nR!RK#NMI&uC*Jq5U_5%u6{Aeu%;WrVk1@~Fv8i3 zp5=m^WRrZDojTK+CsqjT;ZYeI-Om=!rW_u>y9VfC`%2q$0w0kE%GZ0^8;-;t^yzhn zq*kz?`J!(-V8g49NaE4x(_A`Y2wrP|*%GO_LR({aFXFpRxm%PD;o~)U*rYLufw8N2 zFi^-QGJ@|hFB`}Ka?Sje8+r?xB;sh2@w~Y3TOaT|Zu0y^66SkC6o1suixx8C!C7_` zz-BMb#p^I%YHiJ8(X-%dneI-kvq0kN>Da7n74(tinyFxrKGqUCe$vrMWJa+>#P`(R z)>^w#_hF+Uz;HYT1C)$Z*(71)6Z3!ZM^Iybp<G_vm-$OOApZRfEQ{J7gFCK=B!eD7zJ$xM!jp;)O|V5~f( zb*(C}!B~oacv09cjFY``HD1Bc^QBdOAev+Zujqg=FFSTUA%_|gx#tZd3$E4Si;_S5 z2xjCjBMRTf-I)KWjeOZL3!3LZ2SN`Gp?<@BJul5EQS#BNwb<+H&w~|11rvIWaSOj^ zJQPms>RjKaMGqw*?xaHSfjKu#*)@E6By2Q%ah<1R{?25LSL+y9vH*BLcP4kVUcvBp zxq*`=nBD`8l%K^Z@`m^G2@*~TxgP5Y&3U%3ZIg0R)?N{uaE!Tck$o+;;V*`KOXp$+ zbDABrUL`-YH)U?Eb0c{j#|R_cKK<+XCPz5vJdYl4bYY1fy6ng^!Q*DyO*@ai>`R(3lXkLu@2go-;>7FZ6*;*<*4U zsfd7}Kh8HdDBb}#5CNg91a#0lG2! zv+jbqQo9+|oZRt_=)U*r`;txQF<#Vg@Wm=;i2SB*v`o7eZO%tH0&^j>KYdnTbWE~w z&&~dFoxUwIlRHC{rNGSK4=Zv$PZvfsE7VpJmApD{4HS3h%K$mtj9Buou}wCNU6Xaf zXA#5AuiKwS?gdejrHVsjALBBgFwh6|Lv_{8$MDBg9VRaQJahOld>9r_(kXedPQ
560~=Q9x;N?eKa%Y6aTFP{^r^B;+W?2+Gd^WR{%l z8O4JS`u*PtqL!Ikw6fqpvTX%h0T-G!nr#^T@t37-fF7}}4X}5vbeEXr==1#Ax#sRW z=9Fs@!eV(Jo|#5ef8MuG5NkdEWAZU{WTdgWqkSi{7F}-%HL=Bb(Qx6}J2nQhuYPIf!z5jgLf zL9!QvCpdEUw2U_ZG4o;{ch_idQ1|)U^v4$Coz$?Z%h)F=LWa4zt6>O_I6l+{bK1dD zt!QrV!F;w%VaE@&Q{3R4!)z;G!*g4;3>pxiZSNZqj0H{v!7db~5qVf%CyO~gCykIIc>*i|&kd7N|+%u~!lvjzx7(>`M~ySbJSm+>Z7xQtXh_(vFAG{^JVM-Ic&JxK^BK;OLQ8u06dMJt%iF?*f!0 zTnh8gt^OF7e8BtpG@yU-p!XtOy`S)f$LTbfpdqvwoTCTSo?LF!%AAhxtb5OUj(Zm; zH43K2r%bulvLCcXuH<6|K4u0!SI05@PXEn$`K4`R3HvoODZggtwnt;NwL6GBxE`Uu zb@$wU0n24r7x&vn1tBTuH!X)aP^Wr3~ zd3Y3ZkJjiJtwxa>GJtWHS{KD$87Z!IuGaNQc=2pQ)q#)P(9{%o=Ma?;(M1El(lvxf zM~1HUPWBh%rUS^Zr@Fj2d`F5|z)1=rq`MNHI2lI)72lM=NcMpQ$ zsB@P5I56axPBc&Ggrivt2$~i}zZ#~>k@nra!)0{Qq{3iNBTO|mK7B3X`O?;8hBMHh z>h{h1tCl#sh!ZoDuk;Wql;hkw;(^#&TvZ313D9cB5PBC>ZOK87@B2Q;Jnt zEN65)7dtz00-X^6T4W~GBE{()_|Jrh74xmxjiW|YUHT2}{`OPt8+!~e&~3q3EMDqa2UVC7V;_pxrbS%h;9C*fv|@>ynP*?8 zx$Bx4OZ>$XjvYMwQ?`#Tkk zD9$=+jg5_{VRZLB6#^qJmODIvPt;Q)(p^eJnyvi2_>X{nbyi(BfFqh|#jsPlZ*eub zP_hz8K{(HE;5Tj8pKYzgyRmDpq+t*@p=K2`FIQ`t2K1B(;zcS@jkAUzj)t#g9*dUg5}s+cBUKj0wGl# zcrLzz1|-kS`bqzDXe*$JG`#K*yVLw>h}o@+ExxC3VfYf;4&w&%4TzAboG!UPHL4(( zX%d0``N?P$C=xKI!ZjEt*j623sK=AiiQ?&5ZaC7=P2hj_n3i)nz`uxAD&{azA;`2G z60|6Absg~(40Fuewh>%7S)Rvx&dcO~NkH5(pL}#^5lyWwM{x1Hl{76BjKDYR>e&%1 zqRBQt7VP78rtVG3XuRXLugKvBS=n$?)1u@nufG<-h!Q{YOK>u58*5MkPG2QJh;?FC zm~~_~{;>LI97g2Xu>l5sEw=C(iY=1l`1A=C(XeV&^`e^OAfZot+b_Z?0aux696g6| zKat>(KCIZ`J)=SfHOdrjKsbIjfQV3N(bDNG9H{sKOh^*S1HSVeX<39WUk5+e4jmAh zn>yJ@u_Lv-=A zs2rG1X#^a^$S^d#AMLS998+Dovvtrd${GeQ{;pvFox7BPD=Bc9{+jVfDj?VF_ze}) zADL9Rrohevs@su1h?lFw3zqM>=Z#yscpXN5c-EN^*q=dz*fQn5MZ#c3jJssb|5Ln&CgyBCu+Xha5BVZ$mJXKF_B`ZByKZ>~|HA;i!4^$Q4ney% zviH6P@BG*!!}e)&9N51WE~w1{(_aP&r7ntozC9Va#zWGjB$N~{J?dgG9B&C*5oOY{ zi7->$aI10s1>!~kBj1&v1>FIZc$>G*K8evYc5yN^XkLsu1YH;~hVJqy(YH=Jz_tp1 zJ94BJob>?_K@OkyA!i$e>nM4bPTK0u!R5(fT7$$#c^mS4=P@gmm(?dxsA+>@euhah z{KJKKc&7hEQI~T(bun=l@O;n+UH>Hwj_NM}^v+F(eQbVML%9(hBUh#eax?8Y?E03& zPGRe7Jk)Jvupk68vOjR2ge*Q|*sHm}#Ucm7jJdQW&k?z&W7uC6orL1ZKmNluvw)Dc zAzH}>D~T9L(B?MoXpN!OQaAKlm%JjAe%v=#987y#Z66)U;xid28StB+L>8?PAG|X=HbrT8;-L>X# z;FrMq`Z5I1PRuYf*>Sh`i{)y;4Xfpvv~cgU+0haX4MWsuT|71ZUbW$ZqJR zTMBkZ_ZUA?49W9W2|xN9qAVmx@y+Enb+#%nL*Bf&yLE?#@0EeNT)pVUaF%jFzp9Dn zm@W|$5q&Xq`AYO_>Zma{g{GYva5NT4Q4tEsWKxLs6q~0nCJ%d&4g0QPpYGQg_tOwy zMKD()40o%sYt9afCb>4-DBytvBRh%0AaZcEV^BkqJ)q5ltV$j?yrzo`oysC7MiI0M zuS7+0qEug{7QXIYKc9Bmo8`W+`;TXh&3ODK2~bo?WRBj)*(5D^`Q}FMHRfvNUBZ7N z?Nd8km^F>7;~WICe*C|&0EEP&ix4Oo;~|dZ1Q2%9=kZ;XLPhO0V#XGHrGi&EnC9Dp z3(Ld_-PNL<$nwyxugUp&G4Mxs)!P}}2$r2EOE%W)acvir8G?*oGJ6@pI6kqZ^UO>K z4Uc#V6gLg5nwW^1al$8{M+DBSLRO~$$nQSv!EKmAhDl-?ftQRhQf({jkAzMcq!H|U zuvZhF!Zu&I&wt@d>7jy8!9#u4dr0CR({o*b1&Sve5Vw{PHzWB0TKtyK?;4yDxwSqG>&$;}6e7yr>9NgA6+<2m9V%v?=*fty6wr$&J*r-Vw+qR8{ zjnml1H$CS$-}~P8bHC>Y%*>v>*Iu};b*=qS{>bgjI^FhKDQPXC!I21=1k9KceTpL8 z!vi4KkW;tE6~)hi*YV4WQqmp3OjzvDUT5_FLU)c8;(46_Zo?wU^`0rOUyEyCz!!81 z;?X;tt4Z@I_gdimhUK>5&u*?G z4>)9J#rEGNfE(f0S5#M5z$i+fY%(pgV9HZa8O zN558kc~jM2h)7WtaZWf~o5CfCw0x+47Q=K75X=dxBSpd#XTxXQ{b;W?&&jVSkW;Y4 z3zQH@Hzg)I$-5A#o4*h_8$5i{)@9cSPUyoL*^2s-l(GHh*($h}?N3ydX_c#b56#pU zb?n8u6W?Xmx6&&2YYlchge^47VQu_mm)H7H#+7EuY(gVk z;=%4c5@xs1c}#C!aeGolk*AZTh~S7XZTh5oQh*QJOJvpMjO^Cv9WDZ>fI79FJl1Dc z(1ZnGrLE zVi?fQL0}q{GC6R@GC9O{Bz6{S0bS&e#0a`SaBcd+6!VXX6VtcN=|B5W#_pC{OS%cX zrdB9yzPRzWIQ2D1^tSE9Bl;Rn*0cE{YUNF=$Ld^v86lN^R^lL*y!qCS-9~K}S_!5x zo_mBFz3J$sI?^7dnDtIfmsLay6>QGjD(}M7s)Q&l50{qxytv5<`?*Xq-Qn(&Ui{2E zHS-g$>$TEwDMP0SWtfi+1D%kdOP+$lQ$~baZMx|jg_i#5eR7tz)RT1q590lZ2aY~k z=lBNC;Qf>Phq_`Rw8caz7@hFEM%3{DrQ=P+`;v`;k}f&W-xX~9}Pb?__7!6=iXL#w@^ zQ|x(DMRSIRk-)L$ePWjDwKE;FZsEIH9)%7IrGcf@VECD9svMjmi^2-AEu*UzuJ!Yb+#6;-HIjdqOXhdE_J?lcX8J^b{f}mQ zc!?&=c19%xb^Zc_UZc3v5&6gINHlSY80v!1PZD7P8}0kacsVYc-E zs!nxR;nO(d`7N*O9PD%FN;G3;|2ISUhxZ9}9se$m5tblQtNUYNkI(el0`X?T29YXhcLKA>s;-M%$yS?cqm{vrh4nauRZ5l zAS1(3V*x5yhx2!OPR@2@(b}2Gr2uYllSps^@})U4S|00hL)}l{C)d97{?8MI?@v0S zcKdhh=Z>Kvih6AGZ6EtmlSFHvo#G|m>-K)?k8z4$LfdepNIybTrM~c$27)YZMvY{m zEq9@vZgg~qISiK>C$q?)Bn<;vt%Y~1NI#a6FvOSXd&QBI*VoKcqcO=AOrv+z2aQ25iujj)6`Dj>ISTVgwLpcekSJe+rfgwQKkV>$AuE?z*m|n4Sq)pgE`NCshK46Y zpg)i=k_Rqlh8Ip-+;D~X90g;9j0j5vV`-6UEuK0U?X>wl$91g@9k`4CxPMo^=nNSW zjM>_&A=|Qibx!or@a6RjT zeN=$|Ow}(ZE*_3we91A58K=bTALj_h$r4v8oCf1aXzTV$$P@hhT~EOhCwhGESAacv z*rG+2N4EBNDDBBG-gEixB_v<^%f~8hu7G|_-yv~j>hisq@O%C^GwR@E?!2LFe|uN> zka*Ac{8<-wX{kfJvV$1b=pkPMS2$6qcg+i@-9_NJ$=!c!kgdGJ0NmM#GROJqdX9&) z5Zn>mGR;o?lgd#V$n?UcwFKf@Kc$WRpeOS7cBbRws%_O?ZgaIHCkSVL2Qlc^_qsk* z1w-d20Yf-0PN#Q2&L^OKjY^WsPP^ASIp-tmkl=+2=Olj9iVUvv>XA@`2;zj8=~=Hb z>7;<#a35+L^e?WPZLc#n8EA^r*7L)M#9Lq(**(Bb%NYi@(F6s;3)k;y>eufvAQ)jGlVv~Ia^mKhc2T04@$a^=K}2*w0X*ehx@JQ-R$t2htzGW> ziiHwaR@M71H28~S0gD!7A2gsbVZ?pUv%^v0N3k2%7gow_^{9WUmU zyuo7gW>(EPqS(G85V+e8Z%7d54Cvk6Sm{b$_^R`$M=?mY+nkGvKGzFV$~w>QBSgK_ zPh+|ozzrtoOWzLC0eVmE&~JH7e#aMTYr@;~Crt~flV_+nUx*fd>qN9V#w%osR$Cvu ziHUVamJ0S~j203|kl23mk(x+;GD$%OijWu~iw)1!Q0$PFPpI#d4zs>i{(uKKxfXA) zZn`BuYHarLwu_d1NgeSQaZ#Pw<9FwTu!(nV8e{SJQ_&8tZ^_#XXiU7cgT} zMNOyGB?*ms>Ii&WzMVj5$-3rZ1cWhh1}95=Nfq6z>6xIgf3ua8~>GApn3^ z>Rg80Y=+uo2H3{Mw1*J}Fpzq8rm-+Ai@ZFzBw3!?!?`5)(M{<3_3ITc2U1kE_N1F= z`!o;L3C5LTFFUr- zo{|5d*2`iqQrYL^?x82WYBGHhPQPP<`*V0JAfC&-!E0tZ()>7=_0t_0Itqr6aE_iS zPGvCVk=)9$eEek{ib#!utGJ|ZOpgQ>Fc_aJ7LmO9)W?}pJ$=kX-`aR}{;Fg+eteyM z`AAw2SWPxa#TXu`#{7}H55ocx-OHzRyR(QtZ%Os_*^}qf2Z+_AV!<4Ii&Yz}I^kx5 zk}wtD4|WxHl0(FPo1qO0(6VM*w=)6le6{#fBsiP?4(ql;68fyW#tH!1-`tI8ScQ#J z*R@gr07D~IY-$f(RKH>{s;AMCOt|}F%4x|FcyDla1c#-Q#MEAWvk5UlcquC!U?5mk zawE6Mq&bUkxsjo#blU5P#pxNReC2CUMr2+pUcF^g?`NipELews8I?j@Rs{&sP&jx; zb&aJdhYwZYd53O*_=TySiSnK!Jlh>?7{=&mbwg{c-RLZ(`do-7HD=P(71veO$`sG;qyJKz8A6p5jh^Z zQ9|OrCnIBZH%tr(Tcx&C;iu-t;*^<5anLEX&M9mZolOt0B5~|+O+f@+-2B)FG4cNCO*s)O0 zm-ttR4Kn1sT6`590j$nj(cKkti`Hn&=JI|BrV4>Bf$?b1Iy6)B*dlr^CS1IVR<>LZLH~B?M2Rh(;3ZE$`sKs~QWH_rxXtWKS zIm!>Zm1RLfg9SW+V$oar)t9x?>~^%PH4)puafr5R=S2zHXF*@#B|1v|`^2+Up$%{;sAIyL-@h+==I9bm>J}h?vl zf$UXM&JMu^pB6Gw5sl&qyY9KP(bjefW3y~1cpRgPa_)T2#_T2VZ$2(ymQul=G?O`= z%5|=-hBIt|d@8uuPW*UUGjwZrPVV;kdkOT5L*JRurs?2)el?!ZKC;va_s`_zrBHn7 zIHfEfrvn2-%kA8$jeu%T&>T!IJ1U-WS~#R@HkI4FhbzLa8NA&>8xA=|+P9Cf>Mv)m zYq+1(*OosL4AkFGe&R&iUAkwl3~_+4>+W6hRTGK7Q*=qdamp-^;N$ao{V=F-NSUie z<#EP;yW8^h_->o<7(SV)s*owHC0gUG&%-BT|EU&-ek2S<*N^{=KJML@zb!0Lxy!82 zQ%BE%MFp{sYdyS^mkPVS&fz+QP3&?o=~8?0>vSi?L{}4=Uxn;bCm(+xla z#SzxR4u(k6f++QAw&UH=6NVq&W0&c#(g?Nzqn!k~o721I1xB7n3hq-z(+Uo|Coz$) zL#vSb%Sm|v`cba27|+YrFYo{o3kRt-Y&mThs*O9`Hq-QW_+a<=X-7QUDxbExav(4` zUg2vjC;>xwvjo!XenG8HzHLn{awpnXE$~Df_p=u)m6>aFVmrg-*ep85P=A>)tcP{< zpj|hu%=Q0Rup}Yxf3XbWN!nAczp6n67YWaBYQXjEEOyKGLJz#?xz2JTq$4C_h{*&Y z>s4u%`0Jly&tyk8qH0Gs&X;5^Ux#)wm>5`J(?2K>1X%%a9@oj+5@j-Kuj93f@xcO! z>sR#{h){5jy+*Bwh`Nuu6v)V8Y(v~9;*^x#n0k-tGOEUpWr%}al>m81`?Xq$guTCt zz6fpP0|4{U7&$`?qBLMI^u&?l;NZX5Xi>;pFrXjkLSqYGz;D)gHiaM|)x&Lw`rd4r zPqY|o;TBA&k96|2p(pmVp_BUzt+6O58Qfww(zP|btZOzHClzT*vu!jqs%O~62(e#Y zf`(ImUgCcpB(iMAG%w!=g>B31tN%qfI+7`TSiQtE#_KA1%lxu@O>Hc1ekgl$4#p1c zZ5MlY!hXFpzTCD^jVr^Qs8px3eIm?K37aGfWh(^ zW4iT;_`$SpScxrZIPA0#(N&7>C@msWmTavB`PJfIk-=H-hbmzqpL_8-eH)6>c|y zSLr;sP?wkC^zXYc^bft)zZUR!WVQpu#KQs9ewkZ0B0$5wLP845Dw=^7P_xS@A z)GKPtb^z5bPHK^Q=%P^c)4n!@PW9chBp@w%&v?B*^(zFUdk$J=$e;*5bL2pQE z1M}b)_}f;aORr1wIfXs5vE|4@@lZ{QN>%_sd4Wgm+h%4iag9;9fw^I)YUulV{j=^P z5=uy#h4@x{{jyj76{-@yuw%Uyx4#t!-c;LLOkXwgwpK{Z$crT#SN{aNDt*2`QS@w7 zmxN#@@Rh&e&PZ#f>smp$nR4bKJmh^KUiHb%3Iww_QXpY)lV|xRqR8R$6E>ffFf6*! zapm_}BJ_|sd&{-J?&=UU@>)%|EJ5~VP(weSj~Mk{RFs{brT>I^)#-G*jd?0w;0k9~ zZGrDK`ffC^946!Ul?nG%l2KCYBUpU9B*}WVd!uA zg#FGIAs_g;neR{f13j#?qcdYIlXqR%$vsj|LIze#b;9%bp35Z7F^kX3RlQ<_9FO!x z7ON1w?!agzB)ci;U1by}jla;>%Cp*1Bt+_af(nBQRq$U1 zUGo|)gzqX@!*fk?l2qOxY+iD@I{21l9%aUjz6^prWnZCK65Z(iS zh1>tb*1CTcD8c|52R(us`G`@-ytpzTLlvnCY9>iQZNz)+l?a&3aaZY6@q{XZre_^? zY2x=6ssK_N#cnB)>Q9sux+~iOhXpLcSP^#v4#q0rT$C9xX-BKQZ)C!Iw#>?{!A8`n zc@2!hydkN-ZNg7=m(y?VDIiP?e15BpRLc_PvxbX=aqTyB_ZmdBTZj3w!1o7LGUQ$_G& zdJNcd>cZvf0d?-e5Z(E`(VR={y-Ept56xv8AcoyDij6i3FZw?0R;daoVNf#g9s1{8 z)JFf)dF@^6-%S8O!D7%!ZSwzBK{k3845lGsulAj`AEHLPC{5(&0~6j)7Tjz-XILg9 zTEkZOSh?=VrDrUm6!VbrJR3~`IkB=7rAZ9%fHH&iMkl5{E55PBn){=7ZsB$R9JcmK@K7 zAhadPgz7N685v!DK+rkTuy>hU(mDLp;eV@gCU;}YzKkX}tiFs_u$g7m6N{8<8B)W4 z$VzI23hZ=2NkwN4@l{$PVp=(2H9!GseiSoY_DfI(BweISqRkdZ^^}Q4k4g#HKkFRN zDOAF}itSz@4H{kFyrp*0wJdiQz0S~Syl*tPQRklb7a;PM2{a1l=Di!irsiO3azAkb zBH-1&>CqGj6m&AIxM2`*;~NRFDFRq@%2{6dZa+T$T|e+YI-v{Z04X}6m$F+CGsE5O zP{wUMfbQFDNy`-mkm5w9Ur{SmjbdMp2uRY~^ZEJG+P%Cqw-yCSWGDt)h$&Ac(E_39 zvOCA=!`S}B;BJu?FB{=nEjNlDcC_3?;W9lv>j~OQA$thh&jW9{5ta3RN0^XRY-w17 zz1^NlPIQsaB+e6AyT8+)Q_5I50dXIrhSG_3^o!S@@e9|VPlvv6RG3o{%(UtB5XAWIh+LQ0-hDo z7y^-Zz^YTuwu`f5Nzha4A_3L(5_nu#@;q3U%+66uD+wKDH2dSS1I(!G6_6pBk~k59 zab?7b^~D2Rb8Gxx*vH+_+$|Oy#70dZJ|<-zK={SGc@?zdLkWjP*5htb`9n9Jnuo!M&nM zKe9lg@}8qSpKL_PEyO_13*QFLeTM4^8NVF9+rxT3A?v%Zs8^H^7dN_;8KR)D!v{g- zxR|+rkre#LV=;nNp_a^p2NFkI5&m?)UA+{Q%WuD2LR7!I{!#_OFa=%a2`J!fQ@T7o z=WcI=JQIN|`#AcA2l(35S%j!-PGH8GJnLo3A2zO6XTPfsZSBfW4kP4B55z+WQW{iv-OA~*f1g4^NIzz$EF z(E$Ch?0q~STCjeGU(LswO3FHeF2F-Vx4~3h9|I%SZBTj4`GBPsP5F&Mke-15Y$ul4 zOc;phh}H~1pJGwe>M{lE;g#NkWDOEIHV;zO1;N zrWk=qLe9zOQF}!Y5+4^=IUp{s>g7TmMM9F?00O`s9A+pRnOotv z{fHbyEtp|?6z6JG<(O$)Qtr+aZgMEjd60Tf{*ZU~v3Orjf7ok;`zwG$s)G00q5wxl z$f$bUTj2gV5$T4Miux0)AX$FG3$DRm>VxEJK*K-|+p2XS61zK=;8NLlp39S=6T*wZ z%CFnCU;Y%p%PC2*zkgHf>M6;S2zi9V7zUD(ZqJZV^&UIzL3ar2OK%HfwU1IM@Nrpk zi8;S#&MEcD8CbE=XV`u`H%fR|<}PSe)iMCv($wK3VE^ZJDO|!Tdx8nl24unKRg*v7 zu2t<8G0yB3DX4c?ZTYBR0;N7PHKv3s=xzr*U>xvWumzhT5~3&h^9C7i^OFqRx?z8h z|BQI~yXf9%qZo1&8E->WycaxhGVp(ih5rAPOb1*=NB}qtV&=*jBl}0L?>lsU3t?I~ zRUCl_{)&HjJx<-QM3xwBB3|V=0Cs?s& zYQiHNqZY0jw3lN|cr}S7M-%Y*teF-+S}1Nyo_aO$x5};-EmdL0owMJ&?F&rn{T*-l z&!V~^Kd}dVA|(npLE}Uwu)RPcU+Ot+uv1f24Ttvd)9r?q%J3#qs})e`buj}2dpKG4 zEa#|ipI2222au!yiO>H}J*zsh zK3Gqr6uA$mWxuuhk1i{s)~`Id%-imw!@YDMT#A!D2koEj?0*EeCer>;!L@rjl^r$y z-?lgZZQpyX7)1+@!{P4rfJmx}{T7|S{-@hlxLQysEqYO}(#Yj# zH@3I1|CbLc{DiIcSNN%>orlFXA~(>~bHTbHb}`Ru)5oPyh80mcT(fSQ(;nHH)PoPz~L3*zL8P)$;Sn1j=tE z?@btM+E^cokQ^!QZ}8Gz%p$-l0VI`P+WwJZKTb>348zWPy1DG++4KyKejy7I^BOtt zj-lHd^N{_QBLQ7HhyXZpB7~I}mbdm7Gm~GMTl@Ap4ae2r`n>kIDv9~|if{NqO#PS( zo@(~N>(?;9cMd3V8*J4onzcGHusrw24Ygl>*?u9oxJcpub+w~{--KWR5!`xx42=aT zLDo>21NdMu@$*Nydt5&vX^8-gW(+w!(Kp!Wx+1{sp78L5=9vc5N9HhvNR3MG>9S_wK z!&h#zzjbT;-!`KX^yh0K9q)uL-on(7+mZUqE1|#y3-pjtX`FhEZCoBsQUwIT;C?*z zHnpyzyyR0srW+|zIbi?uZhwOVme_;$mh#I{b9ZwlC%AR;fzW&WW0$I`<80`v{1hum z0pQ$P;w@|HTRS8sgHC&-hMvy?&Vm)ueopZ;-)=yeU9?t7J*4kX9#kVtLgjr-ULNFE zk|ULsmel?JbR&z|y~cmG+W@N=fe=(6i$6)$OjDO%)cExfXH6}1QMg$GV~cMEw7HM; zu_KZe8PcOBDzJaK?tOUhM;}eHonnr(hR&ay$8&uF9;D|^Q%M^*qC9RRPjC6ft^G}V z64TZqj>NIj z^_F0=;%D;vB;&w*N@P3=B%OnbFh>n8l3%K#mF~O#o7?%1SZp)^bjkKUbo_Aj0EuT; zGGK|f%F3(aQ(p`2hRY7Tg3bpZh>=vaxBN}Lh@DU=V)bJ5w9O{L18EX!I~X`(Z_b(R z!NAN{xKLa;kc37a^p5c)1>6H5gYC@kXo-?fP=9Iu=a>E(Sp4Pup&+xN^ir4uMu~{b z!$r&s&gD);JY82*yvPA|LE*n7>VK^M=nefeyYFM(Cqw2-)myMz$+(g_0T#b}SdsKLF>h}Zr0WCcmz_b(4(t(|^vCy439+I`gAv|E)TmyCl z@K%0LgV-ddnZoRZGD|qnLPAEahu>s|6#%Q1&kB}^(J8itSR-;hI>_$d=>frBCb*1> zk$YR{-XhrKativ);o5YX$({5}g+U3=gz^jFh~``PWJDDUk2~GFaeRU#)jQhx4&8i^ zWBSR+X6;)eWD*HM<<2NqR;Zr2O+ULrlOneW3v;6j@d44@8GlK6$KiHprV#Q>`w7SB z9Ew%z{h`AiH_G@iSEt&N{26$~0`^ivBF=25NuMUhPKUB`$4#ie{m;vlujNUK7mrFZyyBKaA;2RW^ zHqdwMZPLy@&&rVo?c3Z}i21CC12UR?n{T{~Wa}-J2*G?kVasTqM^Gg;uVEaolqr1Q zd;wP2(eVPuml$88>OOD#UXZ!s*Y{QOl!bnq)?~SBr7||3#CZ5SiSn-~=g$q&RyULO zO?Ak9tAzvT!l)SFj`!6b?9}jWVs?ndi8g(|H81o~{|bz2pggP<0T%+hvrP~4^Q(sW zs8r-|#~K8D9;M=$mZ%-ZEHt?JX?KOom~rS7R~y55_3H?POJ)00C3p4 zKtb^JY5UT;`XIu>TMLQGKg_rwHUaI<>A*r_%bF2f$JQP>L%6H(h|9!yR`o#QlEjQr z5g=DUX{4{@(hkMB%0%h`JKL~3s7btc5+zsTfVJFMbNxGy)hPbqQe%(La9X_`&jhbc z^MVC~kDmbddLWyBa6(d0DWTu<-3yDyr1(qW91Tk-;}90Jno;+I?CAC3R!y@(pO<>e zrpzjPIbEVSZwN4ITKuP18mYq(2K-<*M9}v1*$JG?ivr`!P=WK|Rvfs6HWI=SSh+?_ z6}enz@{mD7>d@A#LwPnUD$N}J27RcjMfOf;!IGYEw!Cyu!D90eYmuEPhtKDVtWAVh zzC-ppRgE$tGKL4tGsX+vSU)5g)hl%Mx`GA*39Aa{4J;+hkK*BMK(~W3G+=EbVef?~ z<}Zn2gG2J>o?CUURwg0Z^4XvKcw*aL2bVFAb6-}l7EdvFmNeuJy$RsM2nJacJ5^=J zE&Q+<$}wwmT$TG~lR*j%=aBM+w#y4K6m$hd_|Wz6+d9X~1XUR>adD{aTqeXQYnHUt z>}ziuTEF9RlqX3`6e801&=FZ+azcQhl67nYb8>ex7~*tO1uYB5#;W{Q!B@&l&CpMH zSXN*&bawsC%VKEGGJKJfqVu)6JfRUv5knp=h(oNf{P4OhxD~kS@aPz<56J)IBEe8W zX_}10gP;W{?ar$oPSt7C@M95~eF`;`ce)T=t7o5~ zA65*746Y#~df6_Gys2FHHiJb_gqvmpCirQ8e-^VQ>C1)x91+6sphcO~jB)$9_~0{q zm|BuT9?n_CsKt~J7f(p(>*|jtO1>fgV3!~1vn8Y$cC%mp{h*QMi#$|5;w zY|x5IzGt(EkBy3EM|EX+Qzz&9TSfCMP8wsB#=z{)8ptfPdeTVnxN6y9)X2_yQjj{; zFz*CemKMz^8Nzd&4p{g0pKleZ?a>hRIHasHlpiwgaB~9>+mxsBg{=;8-_^ru1iF?s zMVx&sMCHr-l3;Ub&ie^#9}0PRn(!GAwDJM$oqneQq7L}kChNM|A^!0-o(!$F&IKPh zxNinplfQpMJe%z6JOFA=qVUn@#C6KEbvNH-M#Ch9#DTDLx&_N#M;f4Q`zW* z_!D`cg#UELL5s4bqdtjWmd&k$vBDhh^Bj`%*zCF^-m&mq7d zl^yUzR+Z$87PF!0bD0}@B0Af;liK_l#y=FjwDcQ%K(>O>?`bYFL6?ORH#lYAV(C%f zs7HiTVK^N2$!N853>RfXTw z(iu6F(n}!WP{|>&uk77D7L7&bF_xY#$BIX=p7S*C1DZwp-&EWKu|mM7z=eR&v9SX| zvN+xMR~ChK?wIdy3isWNSS=&&{Hf9RT~xf&giT_G+B_4a;X=IGW8H%T*VQN73r0e8 zhQ<{tbzGTC0X)o4_dT0hY+^C(JP$C60Lt1S+ssFjFLU@}Hir23qfX{-+}L>*C#!tk zTHA+FTXyXcs*%lmKC#^n<79SV3qwIKTJ?677$7nm$?#O`X2K!|pEYwUrfD0kz`kc@ z4=cW2RB*cf+#H3|nTf_qnAxN5a8oe5q?pkxURWj{2lFmbZA7_2+@qqxohvs*S%Bkf zfN2B?WU8l#_>AUV;RR1UbffUsbdFcrMeYqKkhE8CfYo^|dkE$-jA_PTsI$w2F8Wwe z9#PWyjipoL7FTzfxE<#3q9UNmG>kiX&s5mMemx@{&k40R@Q0E)DV&@B^43?^R+;@?|eHjGJ zlsM}aa1YXQ99=?#+ZTOq2^mvl)S+Sv8Fd|hS*1I}=4&$*L?y%HJR@m1YO-FkmU?_- z^xz_X3OZmTw6a@~D zWBMbSYZn38FhfAO!i08=zddm5%Mvv-*L(G3_=0Pk)peUQ@ST7~iM4l3%o`J|hdrNW z1AF6VwUsW`_7X)*k5pK={Sc~ygs;eY_L*gK>>)o6gWGgWtJJ-RMfyW8&xuQP5#}k? zxoJ%So-tKU296orv(tW?a(jHMc!iG@&G^?FmLv0P$1J~}jOa!U(~`S3-#k!^ZL%Y0 z;FZS;IHPAQKqUde@jE3BtkCR>-hkh)fk?qKy7% z-^g^qJfp&zUyF?_am6y%%g)#>(AeGMxvUm=`sWy9T?s_1muI3pa8!dA8vA`j$ zene|rpW*Z(_i%YjFRYhf!XcP9ap(*&)@;^7){JX0xpPUh3X7LDdOPld!aZ9^BVk%_ z^HSs~HpOm4fdT0_X?x*?c;$;kZxAy(rIsH`(ia)SMAZp<`aJ&&na4X12A@`9dZz7GY8a&Uwof07(u=~e7OEqK2t&{f`a1H~yN_#p9F6*x>lRN3 z`Z9zB#2ygIe1pw{P|U7AnD1YFU<3e) ziS8J#b|TqeatiUWmV^)vZtkJ9j)Wo&8tdox^%<#K3-c^}(O?_`JoNt)`}EEHPnCc} zNxuj6$1#E#hFEQXf(#}ZH}RPTE_6!`9zPnlW8``21A*m;+fs91(Fjc2NqM4^a|PMz z3rxxO+tICK4*R8P2&a15St~(!e88uyN^4aWSom_3p0d0d~0`o zBOdBXLpUeXji$IHkxq6(G$IROV0JfAPz0FX(^)qFx$G6cJP=qTqE)1o8rs!s)z!%bw%CIw#&}C7HcPKox9$zFR|1*rY@NC zAyY-(U$G_+Yk3x{LyzXbb28r(iRRXLr_-7$Ed*+5#ozjnkj)=q%Vr|fzGa*P?&n++ zfY@u7OZ7TI_%k*nxJ|h0Ri*o7GRcqObCa#Dx02R$RHsZ;VJ_SbG%;Rz9#W{OeEWps zQN!5z_GTXUaL8Er(Y|UncB6$kL~XbWx_`=C0($>oZ5*WZTjYNZ71lV!a*!Lm_bV#) z@YAMb7N-{m=HwnR!<5)Toy4ZM0&C$IwDQQs5vRjv@^3}!Pq+iNuUe}~A7+Mr$8uuZhhQrR3D=riD z(#5O350E9FqV{^0X=geA_QjcKPw-I01N1)nQvID>|F5Vt(@$FoSp-apA4vTCJpF7d zLF{?Fn;?P*W+8V59;OaFnCbNJ+wbXM(>>e;!hxK(oCws;QhtOvewziPzvYSO8k5c>=X}!eTTF zo5BO`%A=3C%Z{8+agAyjNv=tr5F9V57twI2XUC8EswhD%h63FYn~^|&yys4p6LkbT z8rP4~+_N)(l6>}Hr3BFr9@+wOb)|G^zzwkY3o*fFw%<*h>fp&66kMS>SRqGfaW`lqeXL4~dZvEwL1TRvtP+9C zTNuux|IMqe*TQ6}h3U$Za^y}+*};NU=p?UiV96E%ZwVkPO&=@rLJH1@2P|&;Dc%a=|%ROP3>W-?gz1D-rZ`RC-Dir;aq0dFNrlpomV zKXp%@<7o)&HJbEwCUbXB$2*e7$VSpJ;EwIB0lq(dG0)*r^r338@hS?po=-)*_4Fy- z%%l>Tbv_14=C^O8GC+6ct+a?pro(V1GFll#gLJ0EaR~<^ZUqOyJPe?HzD9uu zU<8Kn>FFN?x{t3(4cbe+@YqfoN3B0PTHLl$?aV$Zysc{m*~{aX*(_?V`uzfj6?G`c zS#3GSzGv2Nb@!ol`x$}q}LSlo_KEZ$TpDO_1vhSY_dQ`#H2Z&7hM>o+p%a>F;~SRSr`L zrl{!UyrM8)c*H=k)=Nv&P-51D^D893RvC@|EGAN62`JeMitj#>c>hg;27^8feVdOp zx8v{X~jKM+i#;o`#v* zP^XDYDES3eb8PUM^BH~qV+XQQ4rttakJ~?jWRU`(6C@GXCfOhcT?;o3)pAv`z8}`R z4826z+|{M!cIsMW9n&yw^K64p4*LS}tEo_sNSp{nxN+q2AAZ`O(o^~~zn~@y6AMn{ z5#er_6f|0wEERi$fGpQnP4bxw;t)Sky7Ym6?oCpM>b9K$vhn;$nhTbH(yc5Z+BLDEITw&=D;r{hF^v^3Hsq_cEZ zlz=9X(#9556*YO4uE0pyNM;d77$2roJ8vmA1x3k7UPHWCypZWF4nI!k*!Lfg z{KI4WK3BEWS6hQ6r!}2;y;K*_CttmrhPJPyIT4V&eQNY~ZOrj4#}(rnG8~Dl(OOV{ zg!)&%^THX_3*Qc6iM5dAI3>}Lc+K(gG)u__YqtZ%l8HETP{?Qu@~FVBwWYeXWp&a2 zIo15d`zL_JkvAy={^Xos&a$2KrT#>?#wrjk)bjqjVEV%@MDT$Aay1h2dmCdk0!S%N z@0d0`fdBLQdh6C#*Q~aQ0LHJ|SDWUVEzTTfRb`=ohJAwv+Hmi&sTU5Kz}w=5SBYt!zOe5zQRVo5 z@XmZfg?`-yQm>JvV)UOLpr?S9s6_Q)!V-D^bwJX;;4XohKk?eySUa!qPc}x~5~*EP z2rfjhNI6i(T|P4Y&I!n@k zFkD*XL{EjZ({@ggd15kIF=TbNUrt_Guu=Jy`)pZH`Eho*f{%Dd1Jid(k6Q`z z$$mz}lKr$q_UrFR2y``#k+NI5mBIk#9~oMV+fAckz3%kxf2mSJ22s zP*ggzqGY4wenmw~0}31vPqgdp`Z>TmjX<;#g1(*3e`W#v4TK2zAp~+;KWb;u^>aYu zBmG4$o`6!Hh(363>o8IIIY-3^mTA3Oz_>v#(yE6!lS9TEb>~1r77fQ7qOao09zSjO zMzD-)RpXoCsl>Fk$833KiQ3vkH>cl~NIUe`x!U!%$9>MK0?q^p4Ldt;V2TQS*uCJ- z|G~g@dmiB0v>6?Bx1(;CP8b4163{y;k<}mwuT|)Bz9FFZ2^`#|?10@qdh0t8OUIt4 zHir{RSC&1NK*N@YmcTdH`$}wKU7%SuuL#c~ddCV&ke%jM=oScc0saOA3D8p zZ)RZLO=flWWS{u)aP{f(c|T^jnIf3YvC*)A5v{VQ8zg zE*%kWf0QW|=B=?qA2X&P@>q${^gC_2&QA%upH=;ulL17Y!|G3{t+opXzJ6nxSc{9f z?H}lo)3hPdI^`3I7IaauB1c1dv10NV#adQCGiJ1#1lBgkk?ykiFD3@-()W!SoQgO5 zxZtQ8bkTi8PB|rI9sGDT0v1+w|B-|N1C{S@aHG*6lF1~r6@fbuVXro!i;8r3H7J6i z(9lkCbkqjdwFp!7Ct*#M{2;`w%TBtrIYT*{ZNB@h(l#((jUB6gFZ=C#qq*;A@$ZJk zI3xWxnqJd3?j)X~>cJc+P=(D8QXWW`7keHfJFV4zu&2|TmC8%kxKvG({JX9D%pS{) zzj7tkf8btbqyO%`ufj+6S`{qxrFGrV!k*lleR5G8EK22zy4|g)8Q-eNYtnDOdbL8L zjhqW1=+k1bS6)HQp(_}vD~e}z!E~jvx>k?ZChq#RwMr%oR?x+eaqV~&!9NTWe z>lODs(}ZCRm4qAUmME|bE+NJ}p^JIlH-)!pQUDV~(b0y2Ow_6N9c6GIu;_2X`oP=D z{|V@1xC;~)JqTQrX}`1l$md83wBWrv^}ro;L4(a$5H=;xIowLZgZk%!_}_!KgBYaK zK&MhP*5s{;+4f_{LXB0UdaZx|o};ooNS*VXw_KJTr`X5NCNyp5naE2NW#32KhTSsm zF~Vn_QMlHBx`cEi9DX2U{{G@tu>gIdNs(+YE_z^{-oVt@=n-GOpX6*3p|GL4$Cd7~NW7NuTgvq0)!D7?j)k^c>CPvPOHN%W^+yyXV@RMs{HkWsd$LK zBeWtSQ~J3DGvzCA+kGj)zq0&rbt!u^*eJ)BH9PJxVRNuy}W!OsFlwS zwX670fBN)8-un=A=F*-v z;8qcA78*4%)`Q@@g-n_d3f9jbe`M$LDBqBx+oaAn|AGEas^h~9Ds^p&%acB~TY)p- za@7levf^xN>p)LXZ30qSMJR@bj`fchKiYd#zEM&s(VObjzEPXPGqO8{jB(SpIzO~- zH@}447xhx@5;%;uEQd@l)#uv<3(9GuI1ZP)JZx1+8G-AStIc|s9Hw<6HNUN{tcyHw z^*GD_@F^-3m&z9n<6kfRFMQD?95?ds=P{rM++Y8eWiAm8JBP`Oln&GFNA|{s4&I!- z&-doJRnKyN72W5WUL_A*nsbJr$$g$T^sHMyknEt9^9Rh!{ptNBv$r~pyQ|>gQQ{{% zLGdqSt?vG>WTuQ_Xc<~AI!8s^=mD#f;4u>PX5TvdGFu*2o5=D00+x zbPU}5@ie(v5)53qSz$H&e~sDye=WO7C{|{_PZy1WMR+eCH1{ug|NpC)*9}~%Z}8X| z8eBrL558@5|J1<$wx;EYR^<*+xx^J_xc=hysAWXjD@0Q}AHtKIpt ziGOMr7f>xB)>rs9fKSWN0xpXy$xUnv^H>Mj$g|mMkNIp1ny0Sdvrj*ugaH0|Howh zGlbs^0J_VU-5gKdkfy{e z8$gaMa>-w~#eby60R|fXX|nuvzu#(OnE!3j|J865E!IDergW@{tASX0(JoY0#YR6e#4*NGr%#v zw%7?Ef%hL^`H27bh=1u>?DMtZ%g1|30`CD)b{K!z+&@0aY6Eh}O+U}q9ZK_e_8bg2I<5U$mi9Q{0NRlw6u%~vH38(vpJ|Yr-~r^BpjN~AwH$;Wr~t>n zz=!_0c1&36a^SxP?*G5iUK!V+HD@K)n&7bpGmc|Oep9W-vkHp`6*7@!$V*`wmrPCj$~MtNu$85kO9Mgy8$*>T8ue=}jSM5;ipV*Cw;KtwHF zJJ4g68_uZEZ+c<`dSV<;>7IRTfVP-Xf&4i0Alv1)Jqe%|qGnD9brSqqDx%zgp9b^h zVEwvfF)ICQqob1(aR!u}7yAJ)tzT+aYy^nPvWX?KS0TXrl}OI}UlT?N0PU&dvd6-J z_#Fca&7=B_w=F!P@?FSXpc)3~@3AT%f8Cs~*d@Txx51d?O+qSPWgJ52|IT24SjE35 zXa&Tty^SgkXfI0zOz*A(DA^t7mGEx{rUmHF(9rJF>+A75p>zEW8CB}__~py#-vv}C zwY*648*!u50J@k0L6rthV0;=N7UaJs%!&ts)Y#@=mhBz%fUI2#rl9|~-hb)kJ0Qn* zzMnC&*Zhsqfq%yz?3%wVIorDcy;RHMyC_9Q(5Tft<6V9oCB>KkKa*|Oe>uVf$T1R;q5W$) ze0}`^hH&mH(p~raMz{OHZ-Srfe`mGaOn_eIp(4#M-4_i*4p`<n8dgfB#(-6#9!5U=v#UlixXQN0E|5Jqj>I!9CH4^n<9c0UkDW8&z8Lf%cb)NfjQrB>Ilp?BGRG;3;g)*iu!p5ZU0};_Zvqe)`YoA_x%^@wXL!C}>;5y7kEK&gsDstB zkplI^MxllvCz~xfxAfyD{D?)&UH0PRLEr%kQGTCz<~&+RrmGrj;0|y#IJfq_Ohc zP}kKPT<)t3kbWJt0eur?p4{or2jxcF#C}9%)KRxE`gdKq9t>Gv{%I#-75B=p+j2Jy zf7)6-yObrz+gqi?BY;Pc_af;uOHi*caf7;5&#u%Qn$QCzQAhd5jWROhCq!PXy85f< zk~lrt3%u%wn!*XW{=G_cTMOlhO?jqVO{RPK31GFgmZ0gC?f3I_N#jq^1Ekpow@Y6~ zB)z2U9OA>k19v!!V^`@oV=H!L|zz8bpKZo9G|#4vGjX2svFar;Is z-JyED!3=7SvKbeZ_sBhUu&u#gEVmPyx5kD(`E*yzZd)7GZ93YCB$0stvx^6_3Xd`s zF(&R7(_D0CweVC+{PUe1-nAK;D*=O{%DADC`DFL#Fi#j$7KP+0jQ$0tU{Epv|J{DW z*W4BooQg9yfu&4ExUdtt!}7o?b(ArLl81oXGWgGUJ@gri1}`wk@xy&F zTvsLTwWR(92P`fO+-T4G$z%IvQ$Bq<--WKP-BP%;I%pY(4e|*v`vNb>K0=DSX86$B z+s%q`lS`&NvIu@Ed{CUm#>%Y*rDSsMsXF6!-^(|0RwVjpj@|ZIZb>Z&E9HrA6xDZZ z6A)U4xYx{#7go&nUfU|B%*(%mo_3g6K8?`%iqGqi{X)p?M5S7ysllsqeijwxTXiVRBlseDb<4oLDhTfPTl7g z4Bfj?t4Ld<5&NFntYb#;YuI!(WmEx4ljx4!#?;R1?pI4)Y1;WSFc1&e)%~nYNTIxO z=ZtmmJ`xGe-o64G>%6a#tn3@m)-Q=Cy$q)JRc=yEquvYO5pemapmU=#O=PUxWg-L= z%9WP3P)>pnL=WhA2yR`W(?MHoC3x69-+bXIf815qnZjIiLIyXs$@lWCPBsAgIoDFT zgx6#lXng90jSk}lzKHFB8a%nA^&P_PI$?W{d&&j#A->LwzWvn zSJ+Nm*b7g9%UF)L>Vgj==er7TGm&{072MhOuIzj4SiZBOTG2@SwmfS#ktsp@;Z)SF zUSv$Ws$#5l5LM__FZ*&T!lgG&cv|zjZS@qYDU14A z8|^MBLHioouPP@rUowm9Sm>Ft~MVuh3}?e?{L)>+U^bC#3o0COf|j8|Bl(o=qDh*N8=OV~tBC zYYbWKx^Rf;GnPbdYGSzkj|Gv*pY!b9N4;quHRnn>@0rQXy~i~E7B0E+q!uQmpAfbFS}C-l=aIQ$US)Vq;a)MfUgv^5@Lnj8)dMH^D^aV?lr;Dh&pH` z4jhUKEg%p{6xG^>X`u60MUdt8zue zH!uH8PjK;Bcw42C$NOc~tBk4A{3ss%K+F;shj0-bhVNRJ?;Cn1di@JU%Cno_^gZLJ zGj_D~9?)o}?IlZ{^?~EX-hF}%YW_rCC4f`797oriJ%c!=j2hs4)0Z7nthhp1qS^c4 zu*2g@7~a%Yy-9Gk*K6+0<|v4mzK3j~6O^BeQ7<%zeKb)D?a}l5ILYsA(zVd)(Es33 z_sx#bXMXorqZ!gnF07mL2VLyaUJstGU>?#5ejDhM%&ISv(tnMNDbhYl6e0DqnmDrm zf}D#Os!xu2kkf|Iee%@D`^U)I$p!;-tKukeb?e-lPu!-2SV)15!Y^yFX_jU$NUnWp@H=qVJq-KxE+JWhV!a?kZ9U-mKJQJk;B;>9U<*S#%B$ zZ_L2;;0q_R-1@9Ji%FCPdJ_bD@HT+f99yEesKGV#zT13J&78G?5EASB+(MvM7g1c~ znWKA)$K$XXJC%~LXIy+)fFuLjPZwJ>wuD2@3_sbSO}OA`u!1+QOv=FG5iTBw8+oJ> zr73Qo*FXU-iI^9$;BXgg31x&6RpWk#B4?SBj9h7Pe6baZ^#(J+PZX=`NqU;>B3)|2 zLsNH+l^O8Wyhrl!j_quN4o}Ht&821Wfix=RLK>pE_u*Sc=}M|jRZ6OcTqm1M&y%-; zAUOVp0R1bvL6PIQ<|`l+$5-8M2n8GMOB%$>4Qh?g3BD*I#y{$GiXI@xhtqxJCOkA& zUzpPEkS9YetNXdsSX)`%O!iC(Tz%D-=Z8lt%uka0G_S9 zcBy&dvqAS#XxEB3W_LB#FrJ!BY)8DsGW$f!?z&a(B=!OsFjwZYcw5FFZ&*``;oYIq zWnxBgYNw*w4c(_Wo(y*T#D=gQGx zI**eFG9;R0dbc1XW;0t+9S3UWXc-2Jw-Tln>FSnGS0Y^96i?%_AqkI0I$@;MCEB$9 zIS&yq>7NNtfI9cys2Q=SE{ovOFG`m$>6dXvJ`vlp)XPe&ZPr(1obk!iB@T{>o7ws`VpO4?YBq>~D>SYf;P zMiE!L@2F>y7iWqU@A3X&Zn2-Qts`K_gxk87&=AvV-y}QJiA?>4T0x` z$Df0Pz>(~OjT~paBZ6$V5T2xiB8=WX@pvicF{S1~Y5pgBiLHn0?9x5V9qJgk*w{gT zTW>!0Up#3yAK!fDz6(o}kBC@m5S7+1?&CcI9!Po{Jj)0L9&u^jErmE-i27V9RicPp2j6eHsz$5c8{Zz_1n8P?tUEviGuVLX}!*hhwkm!)dv zO`J;O;v4A+w}bFEi+rbrDloLGxZW`@PUW9}!n_#9CM-HVfuCxzduoA)+1dJT_POjQ zl0ThgGZVc`K_ppN#ls867cED*zL_q?K)Ork&R7uK?qS%^Jv@<%jc2pP7aGlGgQ~fc zHy!e{Qxi1kUo;5EJr-S+I(G(p3o43|KC((h9WGB^<&rP+>xp!ozREPq@!o894IGfD zRWzgK>lXu_m{~YrygwnymGHJtsO(V4Y2Rum2-09?R%w|HdMiv;$sNyTbMkGb^jiqK z^8mkwkN1mXF6E-jr4wpjLUy^sd{rvIEpGD+LPvTa6I%1DpQzY@3`Hh9K)UEc#C>}+ zt()wvok!OUJj>iLr*99%<=*x9`<94Lei7N05aD`>9XICqa8_{n$%>UJ2=xK8r6{%GK40vFTB`nP@;H*O_FdPqY>9NlonYcn70Tj1=F*8Ts^P*PR+M;@q={( z(k%s{oriz|{1-zFViO*boP>OxnRF;?)E59LavpO#6*qhEFB!LmW?q5O ze-6qB;ohN=Qm}2{dp+Uv45Wyg7_1C4jTd2##&)zxsBQj^VTvC(`Q%}D;D)y(cnsCZ zyzP@OjON^bb!fl%0$CYH*Y1GKdx6|@rf?#mw=zh3=+Yti9U-|{Tw(4J`fXhIxmi5PRGuG^onYkhRYqPsJdcd4+WpuJLrM|P4|095nPCdT_hwWD zU-dLW-r1DRE2zycNmdtOq|kZcMW zeB20WZeW^4m>aIJ|(>` zUk*+^K^-XkT=0uascIZm)0^v_B5fOfr?zX&T5cjks6a?jf(db3w$6hk*D^L zqoMXU8qT^2ZFYyrZ(TW7%=v|0VQ(gl3Sch8rJMNAFght83zOeJX~NalM57jDkSTWs zSFi@(d)nRoHVxk#*=HoiB@-BVo)BNy%eoQ6neC?_B$FcD8(6jU4%>P^fkImgDKA-+s0ygnLtSe@hf3ki#;_eCO;p0PJ z>=x|DKvsg&24$#&nC*6EOjz`GuA}kR(0_fZotHWA?cDGn8(A|B?{Acq76oG~CHp@p zVb_mMg?`8R&LDKeOT3jdtXvEVJmG-i9chTHsSrrtEmwnXI9vqmdVls_sx=5<(|i8I%)zol#N8}TWS$!H0&4lp(na~1 z`GDa^l#J7`fAOX{(CGLFavqO6!n3~-yoG=(IoPXY6(fpP7&n*TzmUg-? z$741DK2N>9iMe+16xI=E_j8*)N?w?$Mn%6c3trZ{<`7GM-%uyL=U;s~%e%ylYTlA` zEKB3S#9e)GmWs1_D8KrG`MZ4$wqRqgoEEg)IX}zDjDoSQE+NKI8%NK!-r`jFS)m+S z>(~Nc21YlbnA=Y1c=?amm76wZs%9d`l|~L5!I-4kB!cnbv7nF%oSsc#kcgXQ8G>C? zGXXz9-;@$J<=i0{Pqg2SPT0;Y)AnsreZiYihNwJxC4=BEO=v9Rzc0zw+k<<8v>G`BMINr>FOGh6yo83`= zQI9wD4uiPHxWYLU#U&2?Mr_3ryi{o&A#1?s^KDk; zPd{Y~?2{XlYcw1aQZmCX!c8PrMxN4a7#L#XmX9s4mimG86Jkn1 z9K;F76bx2rk8RTYRKDmBpxXI#QaUZg%-*r5XV!FIDR(kX4ZTW ztbl4apB{Mu)A;Gb>pg$|S4zxREcD?soVH9dyi*$?GG{dE9&u+!yGh>HHYefyT-1rM zTlty+NvF0bg^-84KoJ@ZEfL@w++0x05xhS1E5I#ar{lSEuzG3L)GBmmKU*=IY1bBW zfA-RPp9?%~${}F}vQv({EuXFW$xc%bS2Ux~yMsf;;qC%XLgU%f z+_zQbI(v5}5-`^a zlU~j`&9q^ucYgE`??Q#ly3WvOTjPzLXRnDlA3uDVSbx8ew|7Y7$qx8ni++&QDRwQ& zom4=XUmJ(1zhBJ32t^3H|0&T@Kwv>MSl>1^kv*DXU6R5KwZAk1-PT=%nasOnfU?Z#{kkaKm1 z`F_Nf$vJBr6ImDbevfS#<_8J#spu!OTMyk8s4&bP?ZXJF`C;Qno!M*?dHHo19!m61MNtlHu8Q6(^K1u25fnQ)~ z{(`Ps1gv|rFE0W-Q#qlhmUDIW7Kyx%iO6KPMf4#xorzbA-FaTB!P|LvmDt##i9FVD zcwAkch14lY@MP8Wz`fPyl6DBB+%2;pn|+_;1h_h8P%ZVQD8cr$M9jYj{2X77DC;|f z;M!81g?SGnsjv8MI4dWqt0q^VHnL?3NM=bUB=zW!D+uYDJM(OZ^eky-5d(K3Y3J;DGcZN(!z~0SFxpPE7k$TwMI#ODA_R=t=|2pMgt@G73LI<91-W6CyLzFr!KCC;qkXuZ6 zrK0z_w=|4%u7n5k5x)B*_Gf=iNAFiOO3gX`%e$YQKAkLw8h?Q0?;-g`(i$GU@m`TS@WD(fg=cl&?LSC_xn_L%BH+{E0k%v zpCw|lwb!;$J&%LH!(uKBC)n8bd*1=U{^V^+2LKj$%<^kR#@2SUkKB8QsoD?4qQ3UD z{lsb}rnJ5CS$c@bkeoL&$n@mBFR(M4UbA@!%Ee)AHpZUWPLTPzN?2QQrI!w8urM4H zahn5$P$bYn(+as18qwe3(yi02)m#NVdf01&=_%(Dc(kWymwK>Gl-rzi(eEx7Qi^-$ zwbqH(SOv z3v5E|Dw`)LVP>6)*^1HT$?_^l`xIgT*W3L$t%07)X40&RM!jZy(F{AJ_=MJcToI~P z&wfD{CsA(G0vwdwX<3@MxXJDOXGZu!UBEBDf5ETxliQ8Ez{R;S`dW1`YobFg<7aAc zFVt(eSB*CrkHQ^&%g(epqcTq8>x?E6FNI*!OLgX)r?Vo%{XO=pOWp}}*UqX_=aj3v z9V-0DzM|H)m2z9Cy#y5hu=zk=Q)|`6nOtyr#VPcN6e%qqWJYcGT;@XWi0jhpNd8}^>LR#TQId>|%e<5CGL7u^lQ!{1~qJCA+{Ir5a7+qh5& zyy?(hp+trf39j7!$aeF&8Z?0OOzP&j;OIOR{@+TB%>6LJ3B{hl7r^5MRhMUNv+WxO z2V1-Iv^S=Oy$!JmPfMnm);xKqcIF!Vg?Lw`)*>9dnc1wU_%n;M;LjiXbdia{!+lYC zr06(T6de8dwWEDIp!x&?`;^^7xR%%l9mm?+7lO2U3KM4dFxS ziq0Fk_3Z6uyPnuHZi&g$jQu>Gtu>T^52Bt92|qwut`w(5%tpjl5dj4o2Ui;c>7}(B zf+-u)zMZZ+LvV+|VpP1@Ny7s?_uWWgI)*%UZ#km#*#8#T4UM4(Kt**JA#(?xixpp< zv@^OHj<0|XN}xYj?-=x~7pRuRGb>SitX<`2>7?OQc*r`j5-h~EAY=mn*j1a~yL0k5 z&o8h}+z*@i{ixxJ_pDKep_sl4v_MYa58s5bnRcKhzJi!o~D za>P?}NKkrVh5`A-T=LyTJO4&INtgme)Q-+GVBqy9+q5rb&M&_xG<(153Y_$7kcgDP z?kIG+S)3+lO;7GoXnaP&@}fD3_DaIEb3JghK49EShiZTtb4x8(pYuf2th*nUz&~ev zupK*e#h#o>o#}&rMfAg%c8&0_DjUCt41`9{gpyx)n^Apu{#x|qyvzJfH%F-wj=Mbc zD~G7T7L^ACSj{-kk*)LKv=pOnN$BOQ7Nnb4p2qvotcfI|j4 zKJ^-?u`CqNpt*A^!T%J;SsBH%mkyh#S3cvdl?R>{j~Llr!kiWzzf;-dck3mjDqek^ zwYSKu)0UjO))3voXcD^o-7GQJlQPAn;g`#+ z!tm&V?@!_P$c4&C?04gu844^Le)Q0^MI^Dc%zR|*RdjK^aX*z~ zH_0ri;d|H$>2dR*^D=;af}_1_qi=Yn{x$7sS@O&Ea;6Cf+4^pWE($bcQX3`TCqagY zGNCy8wutd|PKO>EMAAWrGa{n|1$zaDM7qGz5vD&Y@Z~&GVYDcKQ!`KBi6bvYpOqcPe%I9RQhWN!JHmm+9dHZ6?m{Kz7-+ zpZK)5KKdu~?cDAxx>I6;Q>l&`zl!CiI{&M{YXfUcPNRm{kooYEr(E*d{jGWoIN~r5 zLwgsK2%gjdFzeG@$%Rw$E>qN#7rPV*7qIi3Y++x+?D={HD2MhQabFG0V4PP~`ec;% zhm~%9sZN!`tocjfeXq}YIP*2cm1`w!#vRYeorvb1NWrcYQ{8g3kz|ht71!x|G|pFj z8&8DKa%H;1&}FO;u-I_$UK|aWs;!CQ&PnSZIk@r@-UFc8*Hh znsYBH*jp|I@5Ui|8qq&$1a&)JS73?3#xy_>H&3ay9f6U49LydhT{KeDRQ5cfLNApUPat zfcrX$m7mF((HaRWFL#P|A_yCuh?ZP%5W>W*&-b(46`HCh>yG;Rg7ZhELOyaXu}`Bj z*ExQ9W%FWBgEJ`mB~3Fw1vMDws4U;xf6<7@jeT83OppiK)o}(g9#4F(QeVt;Ne-W3 zr+lP;AL-E*HzdB@h>bsM%-I`f0dQI#2)=Me2VnDpT zY^;+-sAm1S*GGxg;m??jK5$skrh4B?)YL=AnqAi8Rna%&=8lPt4wL-*W+TuSn=m!u zG~d(3mw$*f|6mm_AlFHmA6}Dh|E!n#njwQrW!7t4^Jys*^x=G4GIY-oSM3NQL|{I4uMu`Cs(mCO)uFFH*3r zt(mgtdOP2wVYnFmIkldna!nzt{xxHtNt|{aZyYioF}Hvg$__coRi9852&LgQfA~Iu zKDzTipcx*&9DJ&oFG!ckox=gfit!?Q7`MiLi{;CpWBl^;JPfOZYCoDau8s%sD zovl4T*^QQuU3|W+xazXyySV3!Xgz-^T95+qM%H2zR$fUS1x{h$sv1nRPL00#V9jU! zd8SmE_iYpX47ZHUTj%ufm^sDE-VVXzP+h*46AxEq7YAjgfW32C*UD=~3DJ^7@A};< z9<8%W8b;lRmx0~}?s}L)xl;WL&pwb_p|x&k?Fq|Gh;XVX;+80NVSio3hMaZ3)kG<* zKK>GzP=D@}*8J!@kE4r&lh>R)wB!Eet(_Y|T=?GeN|+e!bG~9-k2I9q+~V zr4iX;P+pJe+Na(o8wRU#l+(J(O}k}s4#${F7c5(;w{&=mq31P&)mI`~bb@t>&kxBP zoY(^oBYAqu-xIA^yp^Lrb6(mjbvuF+-zWf^kh3fr{B4XV7kAn(U zjA=_wm{X_%PfYFkB*bpNt8>$N2@--JkJ8*j%I54=9^PW9c)J-_I>|seSf+_xZ{*y7liFRWp#z1C{f>N1WnkCS}u~b}Ey|pN?j`?Vc#;eaZk28RUY8 z;A^U|U801GOP|U6-0Ac3)4XdQHU?l%#An7quPszLva>rmqZ3EnqULhtz#(CQaHe}_ zM7qNOMZ{lY^~r<)cBQSaTMjZM;iC0QAlRPOux&n2n;wx~?R)RzT@@*Vb@FAmbic2X z{jyiDYSW|A%P!YUG(dh<)o3vUyfO$phgb8Yd(6Wn&Kgy*Uif0^3Mk@nn_z`y0p=Oz zgLrt3Oy`iD`XD^=sTH$O>$=Lc{gFsw)I*Iw63KB_I|~VejdYTVa0^4xfQJb_0k*3G zN1xM8Kx90mE{}`wi19+{Uz$JIHj|h`gZCQEJGgd_YOaAYIKEHo;bdxhK-+GT@89M% zFESZZcg^*t`?~YUsA@gkR-geiAEu<~ddI=oWvFJ6tLXN{aN{jceI_MOm^-Qx+(kdk zp#J}N0a)ozW%LT@yozp;Z}x^ ziU8E5jYx!?KCf6g`3kz)s?oXQs9prkXnt(wX(i;T@Ws0J$$Y(Tu6uq-uPC>%9&+J| z@N~npO0)v3K}v6~3Nx)fZl2X$m4^OwSAu>zydfi06|!$;j!N|JXypy$+gK=@AEx#l z0$<$n#zf%(bs6kq`}Z0$WPhT)+n(S(8LVl}2lqbPHtj@kr=M`?h=H64B43n>GCm^p zq^ms|X+7GgZ|uJ<-C?@l%SVnJT0HAd`EK+uaC9$3y(YW9`2&n4BW(!noh~2f4Dxsz zVmj;Iz+{X(f%ZsB4-E)_k$TE{DlBG^9O#LD-kxaseqNAeDMHn(!KeaA>*@4kmMZHVQxEGeQse z0QxAEm)#~-w7ecz!cIb3FMy{Jc%3<;mw4_4E#TciTRu5Om)+7nZLwDYF3HK-D~&>( zeHC(lYV4?inBG6Yr+ST`!eU2%J6?!*K5D%B!K-_$23FYoQ#RFr`=vfEi7_|aq-OUf z6zWv37J_svOa8pI3ift=e_z0w!pgC9R2N4XJS1+j)bHSWUVTMKhCz||6#R`Pfxodd zc_crj_EcSN1M;YM{WEl?27xMDSrR2(qyhrQa01z^znR0j(vv-9Gyn#$QFyn3H;di-(Q)qRLY=dF*D+!x{zyobvRBj#w5p_MeEXJ`#p=}y zUK4zMwyt`@-KC*3zbt-EqYLG;`#mXi%*z?$;sjZ7&z{OI*SgB@4XX%Q_$J$_S><2; zWKGBnLmoLKw72x>(>KMZ>y`V+0C~5cBZ1GK|?T*{P@uG$2``Q zQ35S<+G71)7b~U%onU4a?7>k&ADD#RuN2QU~7iv6Qi$*`Y1F(Kw&Cmk<}-0ZJMKkkY;S?$-N&(Mfw zx?8J}g%HEbhl>y%;8B@F*2{&USggH7d3sryK#5^4U8I-0MU>@|O84ywtjrU7NY^&e zeDPhMDbqM9S+j=aeS~QP#Q`YdKFYc7_();GKG$gqLi^@n#|y;#4aD+|JK8S^@U@9I z$O$~R!h&)!xG3Ii>EsIY|CnS@7tmRwL0FS_cJQTD3i9VJviw0F)*>l-m3^MZ%bM=!Y)an_$WUsxOkb~L}U%>NY~*`(e3v zom#x@CsuEP?}``A+pkqwpfeN#(_hnCgdX#Oc>2h#4I-F&JT$@D1r-9T33YXm{b-{R z;}unUtA|q5-8Kw?eOej4@{`cp<#cwJ_v+OmcKC1CGZ>?A9B)2xonms|4h%I`B=;Y0 z%TML*V9$wVms|-evxT1QvR$oqk-$zWpr_peyOfd-(0C*?_wl? z=6G9`LTa<)R4**E+H{enOYz;NFU@>O@`=gv*RE#AS!bmWKJ^UWBm>>M=5N$XsV>s_ zuWaZci=I#dMIp?Zim|)ZEn!GhZ_c8BwX9kQN~w`u0ns|KQc#>BY` z5}hJtYW0mADmkV2x;O1_>HhdPcGm9`V7cGEb-et7d!2jL=vFu{s%MEW2$~^c6EnPV zw|Z}qzIjuGemS+{7RBouX~~o7rYnyE$1=njoRqC9OY&g@U9=-DMbUkZd6+q|0~Ac9 zpC0u+A98LwJcm!gMF=L#coMlyk*~-!wR@XpbZFTd!A17AGjKiV^F#sLPsjWmPD4Z4 zq?`m-?NV>M<8Zf5wqh7ZJ*VJV1QuzpN(wD2gk1Kl6ZR^(!{XRvpxnf~+JU;?$6u4q zcD3+1?I2}S^_6KIQwg~23f=n~9FuiZDH}gDn*Jz@_o?-Oits3&N#bflh1ZM0n&UEu zUt?aV)UrHHc{Sf#O!v-q%q0FR){C$#{2(^1T)&KAX~VoU+X)?^uiwFKwamux(#qb# za?btd>g}8smS1f8y|%Pn_F6feM`-JWYEsIlJu0tCpN8I3p*3-061Qq@9j{0-O46rqbA2)cx3^Xj78k3hp_ z4dyY()a11pltw?3s4hE!NagEPA0Q{kS#9@u#v#s(n8VX*d;SNU0b#sU?#{{SCdbY? z;IV4DxwqABX8K~oxb(^V-?F}cV5##W0jSr7d33e-b$-BY-@pCvtdxf*@6Kxf;yYYZ zl6m{L*aStKgTf_|GZPgpnnI$Pgmyb6QHLSkG{iejVsqIjDBEO;0LXMK=}h{SES{Y1 zb3~>7gC7EoDZ75@Cjy^<0(LDRMLo?{N0H!hRPYcu*8>$a!&|$G24ZHZ0~uF$vRhpWHdgvC8&5(HXv4;qB{6bJuFLThbUfpa5fj@jhG5M$N zxP~6nY;(=V7i6lrXOF7|#f4{d>Y_KjpmFa_t&>vCnF}dIMPsAHa^7~9$7=G0uj5}O zQh$8PzlI_{M?7cL8UHIy{6^R2^GzCV7hq4jpq{JCs+lVEw8Vyv3yaX)2*ddXq4r~( z{(oaM)9C=PqIq!BoRp32Cfe84f|m}c9Cv%LcNUd|Di%QzwqjojZBx0!(wqjT6q?^d zS^yzUuW12C8ifHY$9sV4;0f`BFlIXUR z5+EDPh=I#}4TF`=%%S>Q3gQpDh*1Z?gXO0Rw}fNa>-ed}SrZx-!5oX>OAhMLl{5 zqZJ$-%?{5IT9AhVSL89elhe$1sQ;UHLRoSEOq@Rom79+-#-&p8)?|$3QBd$G3{s=|L`H%S3u8PK1U@-6A5PFpmPXhkao0=#2${0-{1H2tRo%r{zXh{p|H{t){ga9UU=tzE=D_9F zlNh*h^Xfgv;i@YL|9J*3ofKy)0FIO}YZhTlo}g!@4WN<#Dl-53r(wX@*V}0d8@uU5PX8oF+i0H@cJm%4W#F!aUp0}H4#;qSd(7BIg%g{QrI10? zLw6#+^rc4ihm%dn*B1^;#F;cXi@$@f6NUmme?3;qK<{S^fWpEPbS`kt_mc?TSZb=y zI<0;YHj$qLCl$E${M)tuYrXdYw13nO@??d6&?=flKf&|EAjlkZnmUw@Nb7rjGB8W} zcuu6m*h@)&=hpr6hKhgM7ZwQs+0LN$n9*VPGX~c2cP0^66ba21J|j4Hixye~W`Z4@ zwlcR?l-~T>#>CP^0BrbX=8Gxpo3x&2t(M8piHr_#LStO`pIA+PewVuR$$coDAv)>T z22Mo+*x&HeXnHn`ern$ZD|1fUqQ1ax*xcg(>}dX@0q#Ix%sP~F@M^>y0ke6auXzx` zg80dJj$AlpVgfvvpLU60D}H|6shKb zKmK5q_c6MWr~^nZyW$#mc$v$*yk~GFY~66reQ{>#qN^%bpIJ-Bl%f37&t<(T=^O=p zIz;5K-$J7HB*a0~eSe^ECGY35BHK%9<)e%l$LTkdZZ@+Lhtpe%_!8&ogH;exaKm*ua*|7)sZ{?|K$50Ob* zPlm{@i;`>r{B@xyL>*XI?Fz;XrOi&c#`-bn(F0+fNL%rebDmjADVeDd1@Aiebd>cI zNs3GLn6`gs|40+1n$xVjQtXnrjN()(pLU7~Ev=O*Cjh^;I)z+(41*!`Tc)93dP;s-CZ{JZDxMooHO&j)AJu(*M73HB*~qXk0jx_z?A&sf+`H_t-y^(WJ90ed2BJIF>FYbrI&vUUygrOv7oiH6ITk)$Y=DRF z32A^r-;EX|53I-dFTr7k0MaY^n>4nKzY-i#nzqK3aI{y=p5cSn+TPa->tL_vmARU} zdd=I{DqRa?kC2%uB)P==7>+NCMXqr%;x0*}*4#sqC$b4si#hbz;w*NI?zK$OK2``! zmJ1L?%ft(n^l=mo2dU7YgYYC@N=@>f-CoTO4ru#L#CuO$re5{ubDZc%Ex+gbelRy`~4e0}W zhaw5~=1OHB6>818Y(RO-RKeEk#G~v<-r1A*%XD8~>eNfS@#R3@($10htYbR)N$kXJ zwvS0zy0LPm$|t$CdXXYZ&=I6MUR{@rN|-Wm4%&V9EW{WDDy7N3>10I z5_Nd&#_66>nKY&t9LJ7!lTQ+S>QT>v|U@s?D^kUX8+$9 z{;2?hC#3KcABp3BfEbL_SW|guX2+7LVpH6IWI6`(IPSgEzTI^ht6^sZ**CQ^@%J4= zs?4qroKGAAwvs&29AIk3SInt)Vacgh-I9p4{1waHp`hEOmJ7=>vx0(OANrw9jwQYimz5NQl?plCVBPfyMoZIH- z0=uWreY^w)C(NZm9{d!}kSKiah84qJvyZJs53w1~ep<)gI@B-KS{SrQSJDJi%BN{+ zL=O}};dKj~_y3mQ{KufC(*iBY@V@VGY9ByDU9lB)a8Z?7v6tZjDeGwcJdlecOM!Ik zpyh$4-1*AzJ<+kJstxXI(rdXzDZ07!WWlV5@bbNuSKkT~_1q)5Gd@K5=@QBJk`;Pi ztabr5><4ZF0FOX~sg5-~iC2e9g|sdXR*!R=oScs591pgWh2Bq>S5t<;{a>sb36EpQ zbC1wCcapSxm(e`-N7|Q#tcgGG6sA~MS(TUWIt2;v+yS0ft5i5vCFkD}9}bQz^#cy;T6ULq=Aa9l9@cqyhQ8@LP;-pio}-^Gx3g8}P5gV7O*7o3$0UPHX92ny zQ5m4;jeHANUMCB6A~u@x7M5T4&&DS%m-t8Hm&cn@PWU?h20B2ZxIyE{fXtPzz%C?M zf4SS<=$S%2IeFP9oyt9WlqcSsAO@l?YoMKBU!2pht&6PVH}xCFa<6x4NI~XmQe|uJ z5P3NdaINb22Txw&YxKhb*ZOla`O(u5Bhww%9yaX8q?MXoEA4G(IDmL=Re=b?(e5<*A`^M)p~$oqkSEXNPl^znp!}blG5rT_&}cDmht-Fm-0h zOZa-l==7tjgsJOcc8#7=e3ILl^bn^UoNc>?tVOBRp;xQ63SS$Co$mN5MD_jgLLS7R zV>`rPpHt$RY_p_8Y@=9=<#&UjNQc-_z&UMTB5|iShn&MqYlGM$(Fv~msjFzsB0j=k zV)NZ9*~y9&)`5c-HozQmBLHHxu-P|A$`EOPj=~Sc66A?y8j}&Ebg#fH-x_+%n=SU5 zgXli+u~c#_pF0SqbI3J>xcNug-90ktcf?=M1z6OsuX|dJ>RoSTU*d3$u0QP!D1=(( zoYExD#L~~IQv%j1t7@HxN4K#4H?sW41fkIe!WkH^uq*^`c%xR5C3(6v)h!mQROQ840EKgqBf7HspfT+iDu8P;t2;mDB$#5OWEM?a3uu+(06n+|iD z2s5;e+&FwQS~!rnN#RsNt}Cv_e*>I?Pi-|#b$U8LXZrD)zN;MbSK#N50$Lb}$+9Gp zK(-YbU~}kb5FBK|c3gvS@b?{G!E$1QUdk`kFE68S(+0c@CjtXB;f$o$D4{vCevrV*=|Yc4U6M)Ka`Uo#Kf9ZMOy2|ijLF{`YlCU6@25jL`S#U5b#JG zubX3^JbIa4Nd*O>$?C@BKOeUlfcQJ&)OGe~;Ehh#$4!S)decTeIOedfbAp~MP-v?X zLG_g-Qbo3d;FQ*YRzv?;`4IO?{*b-3eGZ3)*b1vCJzug+;xixKUuFe?b`TTGbY>K; z!5eE+sg}b^KzJ^5HU$FK& ze|b>cd?U}2eA{V4cCnc>^u8H?=5> z6L{Hhh1xN1;?T36ukPF7Q!VW~eRu0&r;hE=<*Yh%u6wBQ&>$tIZS4$87SYR~h2to| zzxYn5sSTVJ&KFJDy`B(+dm{sHr#|ETGPUj8yn?%0UicXLV6to}r$53s{_fX+zE7xmK(p0M_iVRYuK;IzV*y*%=B!K`$)>DyO62mUUyBb zr=ygF;$S$GWNjG6FqifAT>w4BwYat{dESfMT;z()jYjs$Mx~bdh7O!d(Eja<5Xj6uyb#5 zsja}Wi}2Q&KME+yYdh;!C!kQiu==625jSq+K1`R68TiQ75pAs!xjbilVkUTM?>zzJ=gLyt}Y5JTvan{86jlNVGE3dofzPcyM&=aK|h~( zii)u2o|UHEnGQdHzY|vmJkUO*##jjC{C>(Z3yyB-XLM(}JEbw|K7)VGwA9>taeKuY zKw@$Lw)mdOK6nj5yL%jPJv{lb|7OJb+_Hv%#kn%T`PXg80ceY=0sJ@~c0%Dd9yb4; z>I($yM+$eC&LdR;J<@fq)2ZkPOW$ebRphC7vrzw=pC%0?FI_f{acp~V?z^K>LU0S6 zCsf$pXRW7KyR^a2nJNAIP~QR9oT>ZmszJ*%m(gWD*HjPNel~etn-5I8HCHSn?3X<~ zt=lqZOpN)bJ`$ibdb{&$1Ez8Hsqwf5rbCOKi2wGX$y6pmHkq&5Svr{A1-|Q0vDTi9 zjisfSO%U1?NIyKNP#wgAH2cdc1G0)Q;{dQXAZ|F>}QhQ~}i6P?aH6Ad#i1#+o* zM%q@XY%iuG_Y-M1u6ljcDTo|aQ@M1jGj zo3+yQ>5^{V%933j#JUSf_t(SWpxaTpI5kMG>1n(Z>INZ(1BNNyV*q@w=IgGb!K!8@ zpQ7ZH#NkEOJUHyj0tyhz>o#v$t*Q6*=l0>%h33)2c&z$Zonwf(U026-a=)y)oh1lC z`toP{^gUra{!L=87?j>32owYHo{pPM%c#XM?t4ACf8}`$p-jE4{?R6tvaGTHo=;6T z_KX~gv@(u>0*~ggbZ|R6HFe-nrlbCm_2;<=(a$RVu9F(KL|gZK1Z=k<*yi=^{O^~{ z+6_AsuDpWpY2=>MAi8H=m)=!6x7+e#&+zNUa8*ftnXO1jeucH#=aC0pkv!GhrKG&q zE0^MLTc_qMDMz|g@it?aV|!OAwJAMi990%wxRQf_U2wzuI!#-$qDYRrs%Bg_h36DI zQnWp>WZdPEGcw_DlJ?EPC$=I%rB;fqz4l8=o5F5+C~A^@4kK_<#n;y1NPOnV;dxv6 z<>QIt)%W#5pUIbXdRJ~~Zq!=3+pg0p2;fT=1Wfg9DrXZxL)&LC$au9!=_C9j#}5tk zx4E&O98a{bjK)h^s4iEX9ky&!T1|=6_PBHM$1amGo9+lpo^n38pH3*(3~*caa{km? z`TaNSda$6Z2&|f2k{)P*eGe@gaUA{0@)shPc=6tgoNm&DE6i#IQYwoP#5{vDk*9_p z5%;q9?8ExIfO_@w5b;+<49`?_I9J9=JJlD9Mwy5X#+cY z{{i|%aRtst0^T2-VLc8Dg&lA*nq&n{i(b)Ae6PZ|&soU7!OqE)Y9XJqzfHVp6} zI#*6*&?xp)fgJIAueJA|XN{m$^&F(_x}+J)TY5%pB?xL}F!N3}0hCI?J?lFnFRwl7 zl(e)~F5zjk3hoFdbh}!c%N=dr8z@HYE)Y*?R*Bm?e7SV4xqsPtVJ{o;gcI8y7;MF8 zO|?0&U%_YEN?q|vedecm+copF$WjNu+{7M`blWxyLf^t7Y(Lu8AXy9=?QBVHa~c)~ zoO#5)r1lpdRa7xpZ?_gdQnp1-DYKJ#4SQc!5|6Q|gye193_AEY>8^D<0tCuY8 zdRCnqIE#FkTi-!x>uoXQOa*A==^}ac!2n1pJgcw0MEu~{Dt{#qzUFMEz2^)sA6rKJ zapq_+tW|X;v$6FcW?Hk1o;Hx+SK#0JM9+$i(~ z4mAD$P2%?7X=$?z9_D5xc2+G0B>2BtXz@$*PPD`dp#=gS?@lKb_v@!+wfvSChhL{MNC9SbSNRp@VmDdTCwa2BA;sMN#$k3Yac1u>gPNaCX*<)YECEGRlc=);XwdjEk#^*cIWy9+Y+b57=JE1BxQL z%CXF{-;gL7QGkHIZg>8HR*z7&jpmRDE&QsxkCh`9Q@^Ib=$G0*Y=RdqB zs++GU+W1e2|Lnjx0%)_q(k*LQp-k>KuH_ly@t0iZKQDzyfe5|FKX|r>FG|1r7QRk} z@fV5z1lnPOAh`%;%OBo2Hs_a{;Q!0-%-}!@%Y0iE8UkHX6ygS${1>^kZ*pl{%iUEn z-U(eDvs*;}?}L~BQHoCS4Xz#Ek(;xE;F^As{9gdfKXdLx)%J?CAUI>7u=QW~ ztp#$;-*Vm2o{_Z4?}Q?cU2y)3+>tlJapsezT{UZP*pceFVi48cfB5zv|JEtJIk20E(`;m#|jNQEvxB* z6uLnt<^6koiN>8k2F@7m_?-+2nBUHvh<{Ic?{7l;c6REj<5y<9NJj5o${#lyy`n(}u-G|!zdV;)Vx^;x@r z%h(FS7b9|@-(MZonrIq=Jl$|q%<=b{*9p9Nx@s!(#{kFxUjPn&@9Fi|T37mW1?9Py zS11XRSb;bZojkvOL_NttC~Im&EmTX7E)q2~N_0qE{|{hYxf{d`p8_C-p%ewHV5!l+ z|FUi)(#f3(JO*G+9Ejvm>x&&ucf7r7y~`RdTU)hc z^q>5w2lnlHm_r=1?Yrpqesi5YY=tJ8rxJJN*X6*Jf(*LJU$g4Z&++l^3cLP~61iXAYMp$VuN?cLF3`l+m+d+7H?97wj{+lPp!P@b z8gUjZFi-+ytQ zo%MTIu#Mm|6OynO&Bc=$xDuY?v=}(5aQG7CxA;=-9>~`JV_c=Y#cc|*o%11pil}Y* zCi!rh)&~C1zW00Ign|P8D={Z_3m=x=>i#R#TO$Gf4)%ZUum6{~Ao%oaCodb+IlIgC zzb~x$ukt$nhteRUAM$Y3msx|1UIVss`ZvW3|Hm})rEg=eG_=vfIR^9{E5S0(pM$`U zA7u3V{O&vIBv7#}^oT_Gx9WL+aFh}?s7equIs+y>NMpgmuDw70`m_I`%$owdeDyz^ zK??lX*dG0}0z+>K48b&UImd#&W5HSeKcd*r_f3JlZdaPPHw9o5BMtr!1(@Cx!2XPB z3sRuVq|5aGQb6`i0Y0JnVsDTFd0pETe^%h=O@X7(CZJOs=sRYNBB~qsC9(G9-cf z*PrI8pU!VpU61tQN~JK>1&ZlOyH51}Y|*5*7)`*okANcq1vixcw!y#R_wOf&IX{rA z=H4->egnC{-xU1MV=Z?V1vIvHqm-jE40P*{)fe-wIk<)C7^EgW#l8wz*%tls3F{dker>1H^Lo2a7xsXVcmnZ|n zI8Rp0pCaTh3$b#NOcULQR%CEa2lxu2ty1u(wF{~KZ&#Ce@rZ|;or|k}_eh8+f_wgW z2AJTWE4V>uMSq$xjVVB0URban{ra{bC;j}v;ZD?wd}-#|b?CdZm2b$u>A=YjHBDN+ zxq5rU_(Y*^Cf$nmgx?0=MTpy1e(0+ z(}DtdyzXaWlNGBK4xjN46XV9QZu*lGmb$$0`XpsEi>lxSS6kPpbcTp*}2~wu-E_J|s^+ zwJx&AL_4R^TBwtoh>{Ly{T6G>=(ez+Y{&1Lc@ViR>2?g5}EYi^xM1<8a6v4{*c&h z@lUv>;>i#F+*M%J!X}yvg=OnGD zt+-iS-Y7dPJ4J$Y@c1EPjEk0Q`ipZHuks6*Zgj{tWGG>o~IL<)w*Z z(%~*r%HZ?F!t-H8sIs-qI`8S|^_ovMp2Oa9`7@7Risy(&FM4wD&k|nC^IHHjERwaq zI01fCzvU$?iY`1K8>1?^6%P=o(_UlfQe1j_lHDkEc1NEvjZj z8bz>JC`NZ`1h$bOn}5{G1cGC;_od*!Xj;d%y7QBd8u~H6J=&JW+Vf^`*wp&1|J>M< z;akO7hId3>lHA-a@2hz^IfnOAIe_GWQORG5Qz`+tSuOJ-pSaJo_G~W#2sAi9Ab&q? zq&ZbSx-SZSurV<*tjVW4cIpeEm~xzd z5;W7O%ZIBA#vZ|N+FdJ0-$zAVYQ8q1_GBjwm4qXJ`g`xZ31jpIjg*81RssLW@F(vG$_jlOHI8}(7jpXrREUpCSu!e`66TTa zVU$Z=?RdS%rxu9u{bKEFmSb2%>8A@EC6zQ=^sLFToPvB_rzLf(#-)$Yi8TNv+Zu zdJnopgx`Xg!Jy2Ev&2UzRXRYk-;@x2t)N&X8s*~;TUv5sAmG=bmH@J`i$5J7Kz8!FUuFt4Ad?km_27c0nV6^ro4_Q9Fj9&WE zMNuE<2a}m#5N~8dbi@YXsfs8MxTw!BLSKqxR^zoH=VM}*4(VOrQ5DN?d46hhJ4cMt zuG2hfv5LDL$wSe^l>L=c4+SO|mcxP^>tu7x?M~OGsMUnb+(+LxH{Ga0M$~3k4he~7 z*!Sx~O`o>cj2as_v{ZPxTN&J~1<|4u#qbKm+qK~l#>nj@BjWKrfefO4M40x|-yjF! z0C{FuSW=)9Qyy-fmt=HITQlEepQ}Kt*r)HbC(lFo$I+x}z#V0l7^^TSxrWlJvv~@3 ze4$r_OF|iOdD!<*B_Z+x-o80NzDytinhIh16}OUF_V*KS^~qX4c?^rJBI!4*^09$> z$OsPB9Z>4F>FW@$>sCLwNJ5PSUlOSL${p~ObmSSNE`XwhW=~{6!Ze2%Y9V{>Rf~I2 zS=L(to~zi{C4sjvMH5XK9Xm+2-c@l!Q)NM_PWVehzrVmf%u?0=2oVm+aDFVrAZH?2 z_T${x@}>cVo^j5DK@9Xa^PwN_##`7lrMon9U7` z1rpt4LqU2&21XoE-sw4j)xRx3tdSe+-yZ?3(r_NE)SI;dYtbaf)S$m z>k*?F73T~`#*gO4_gR|mdRkkj#!GtX=PAC$@$ET|jJZGY;7NpcB2hft%o&e?EjF|C ztTrdrPIRQX(9R0%!ALpDNyWg_ws(&sb0X*zZ zT6aBXDGEwY{H(LN^{Gj?_Y$BUG1LKRJ)FBQIjw>EBPh#*0?Vz9n6&vBQJOeDU zt4){>KCV#KBUcBXZqUx_a9@Y*2|rxNWX7Ci;LjYOC`q26wo-`S`oPGqlo8G9BdHB- zReloN1OKK~UR1?j7WQKcw8!^u6fWNJq9)#4{Vu4Bd~n`p%`mPjr>FFk3`dTKo#L;7 z>I~qnH)b&Ph~H$EzS%y#qZZb|IT5@~%>#Iq;U(!kh{9}M0D%)`S`x+~^+`2$@%bl61#ER;F8D>q z5Uut}%hM5Tz%PMMR>$+V?{D!rqVTNW`S)XJ+pCTdG?hlW<{Ym=c@#F@kkzv$^%mKO z66{YevOx}7L*8uBXV`M;p7x4kYucZeybSv15ZtrzwzJJ42vbr$%G|-JTzEA1vEq+P zyza0!!&(#W zN5aO0DYWUy;$tFh*%;{iCg*C<_wvkP-oRLg?mc@ZvOH#{H4Lgk<1*X~o8fpJg__f) z&F#81Uc$pQCTix0Gxz8*sg<0=FGgG!XgFu%p~A&C$I|=gYB*OEYJP}LQbYvFSk0o+ zo`=^8xHPjl%ilm9$ukQ3R&8~BDcxUK6BA)tQk(#YXS|Mba{4Vi_9NDu-p7Qml+s$! zjoioMDLo{!nO-DNna8*1<&awkH zD+&|Fc8j^fJBCHY9oBjUh7|h7?%GxtD1qSl+o_fnE0aG5&U$;OYt>3webI2Ye6~_G zmT2eBoP!c)p$?I2%?=z!E|T5)^P`+()687Na~@CDAbJH3ZmzNvFNUDtDJtI6C1e0D zBUio;Ar{E78=?%;Ab<*;7S4FuiuW_7GNEGZ7@Y}0jOF|1W;Xcw7PvCp@YlqT@k%JcMsP1UEzL?)5IYAm3k$Vsoi1QelzdrEboPwg04e16Y;W-t)-j2v~08Y`1f_-1_JPh$z1%`+Ut2XStQJv4|a2z^9}C;&4=c+YG#8%qn8U z6b}?*&lgrg7-66Zf0U%!ZBzw=TH9Rj^5*astYqe>!{&M_;MOQ{)EJR)_PMECxd+nH z6~Gnpu~`HnWw(y+aoB+g+Hr|w4Ub7cdW&&AK18L-6t_JX%>Jx=+nxARljLGeJeNOp zCf%?C)Y4$Cmh6UCP}Ml=zI%2{I`uEQB8AEwjCsUeQ2rQ-Hgl~7a_=%-cJji=*%PM} zWp#01`ys(0bpcD*iMKD&fm|OWX6dGRu8;hEB7I(VM7Og$;eg?OapYZLE%{K-qiimi z92%l28Ho=lYyCK(p`>LB<}O59xoe%K_Gt((!P5TM{=zh&Ur|)MztQJ?HnvcuP3s)3&yvl@G?3ZkZLtqup5YNWwU7?XvkckY z9yN*qnb-8bXiBKQ5k_Z}KQZyPpeG_BO0u=O#^7D#{wu zefDT!Vxqmz;)tr^5Q)(Ywtla@^W#9v+;CdySespqX>ww|GAo*~1p5o$fwkiYlhusS z)@5fRs&6@p$kFP^p2Jzz%1azrAl5Wo$J9*ShWe`-H*tF9$&b*))|Gd38vKz-=-n5y#RIcX_S5B~OF9rJ zu5J)6X;+~UN!koM`29cAhzC^EZCGzJ(_$7#W%5gZP0&Gjj}AXn)<|i!5iYzf}HcR8{E3x6pJ3x7wy@~)BgL_tz{><$Vh)o_?(B|TGZg1>E)OIk-z0l44f z`b3-KAJGXss!gj^cuWs1_#J-lu|v|kD?#;-)!1g& zxa?R$*av!+lVQqS7AT@GT1^Y{#;hH1dtzeA;Jl=^4w=u(vqHR<7(T7tLbGRzqnu{* zAb00q*X*!0_Zj#9o>j%%81qEX@3SErXM&#UbLs35_FZqPJP^^Ij_v>UGOb^%JNpb+;ljO?;ULscjWQe#^P~s6@N@{ce!C< zdZ~l7ZMDy3$@b(AyXZXGN}DrL@x7ACUNa$hu7`bmNeP+0yEgT^BA8k5{Y?JMHL85h z<$3Q-m}5v}qeNoB&=#BIDm$*_5bH0Wlqi`stJjH30gQ^uVSB%#Nu8H9`*Sdp!Q%CS z_bt~6Iys^Q7m~DpcgYl+JC_NB>&(sdP|KNpf06`6Ou-y$FqjEt_ZUKNKv|YG-Bq)N zrpp!lO)lGBhQ^J<0At*9>>v^Jb{S)+JlR9mg142m_bT7Uxrv{8lt28Nq9kN)LHtUF z{#CJxN8tyc&*hmo05K@4Q+ ziWzm0V$DV*f=414Qy&DYN5LEycD-R=Ho@YWBY|TB4l$ZLp47+*lBnsabB|)pzGmhP z>kla4=%^Vpm4S1FVLiz-{`EsfCy%UzXVGWD2TR^XRi`Y3k=B}Gn$WcRS*t9i{0XyC zVnj669+Mv}`=14(j7Yf~Ql2u#F!O?|q>qeYx6)!eWvWyNJh5k|TlU+HqH2y%5ArVP z9|F&e!mwAnyhU|@YpGpM<3f08?y;0%NWX&}DG6bhb=bGfHHOb}adNsD`7*2(b zvEwEAA=5P)F%H3E~20cX0@1b5fU{uwQlT}vm`bJh_HefF*ba;e%JpN8v- zY~BU%Hn}OX2YD1zUn;>1|7)zSfAS#@b_?4TzXui z-$kUZh^x`&z_>(Zq6fzAl*9AX339uD$zTgYj zBa$kKoc5DXJdcGo>&H)aBndzdT;1Bnf^~KMvV#08+Z>^qRV z4alW^AE=zhUv4qC2&Kr0y?x`G{9yo+UAaq1=tyWRZg+!ZPZuAF^W-xIlE}ZvU45Ei zZ!7>(&b4n9#p_K4j9zbTPOxG<3Kc%g9J~~hE5=tVM46<^ z6?SG$v*l+fm<~6ukASSlZ`B#Aqu&Ab_#Du+J~O{NwA>&gdkdR!M2mk2<&huogxW`kGCfJfJKZi`RK|vftDz&E<9=Wj!Ch-p+9D0{;8l{KqL$;oa}3u z7o#g%_n^2G28z+Qo?~IigW=}O9A}_x3LiHd?{FHN$uG<6LGWL!;3G5d=0Isua)K~U z)VT)s-euMb4cS=3I#$celmuVt&}qb9Tl|RK&%nyBr3~VdENazf;4e4X8_2y)Q`aFa zSPtUC%f6c48V`l=?+qw^MV**JxR};e*)rGJe(n2F%0poT-6R`WXfWvK($86i4sR~* zXEoj!oouweC<9%g(BLmsmp_#op2Uvh4)YT7{?GETh-a`!f zrjTD1#N-z-%84f?SlAr{O6&sB8zjT}rd)iV#eHLWvfGl67i%kv-_Doy%W{(auG>vG z{R}_Vcym@UcUmhinx#m9RVs$8g+D>ynz095SxOa!lfgwD!)5VCDVMq?xL-OJ447x! ziEPR<-R|z?J6Cq?W73IG>*)gJ9l}W}NA|;$t|w3;Sz>LpW5}8jM6I|Q7_pxdnTKc1 zE8~@Jx54tFHA|vLJxTlxrO}{W1Qqt#>u7{(UZcl#M}F4byUdG?!wym= zEmIV*t9Y4R`qCQom7Ih;fX}y5x>s{@8Rbq!K|HI}IMQ+x_So@z(ag)I2@|exwlnNh zJRV6W>7y^bHdOA*``B0$6gbiFHB~y+`CAWLhNBZm(LL$i6MSTtr3o|H5~ylV7be#; zMG0ekE)j&0sCV@gk9QWdc2OT$o#b)e#|cpqG5#o}6$!&MN)75+Owq z&u4SZ(0!Xv($ZOHI z(}!}2HOhMzWR-%7YL!0jK$%DOn4_6{S;Izt&~6aJN919JkRV#2z156l_DQg@2*NDQ`qV9U;OVcm7&Hz0B$#w-J;j(9#Ilc`#PBL4h7briayjKWb;9S#Ja|vX z-n1IcOCLUMs}UJzujQqjF;9Iq{*7w?1?D6QixnDq`MWQmbM*#uSY3Y>JE(~Q&|Q2Q zGEO$)O-FOXY(5FQyqeWPIk7eyyQ(GDN*PpTf;Y$$`fI=gBWR+Q=oOaFEgAq7dNU?5 zj;LL=9&1<@ESb!vJKS(d^A9Y?g+f{GCt%Ija0`zBEKsw2o?c7zLn()c{aqBBzqIUC zs(Rtd-pd@QK^oN|x9y!A{lKu-^%RWr@R8|q3eLygjb4jLRW!BAW7oV*s7NqcsM`Lv zJPf4+#U4TkF&4+6U2uESY3Cz5F>u$;LD?OwuR$RyipLMswqanrK?&LST3j zQQW5|hrg((B!@J?0F`Ho8Lx3||LJ$*RVLxZWYM1e!WcPNwc*A9ccvQ2FM9_9o|o6t>KYe{vv(u$L0Zv*sw;X5)78wV>=clt?~wNRF+4pHUX z%w`O#zyjs|HLocBXYwU?^P;Xz1g4+$vrBs{$5X5hq&3l6KVi2DU5cH#qwYKs?!=sE z7{h960SS7hY_~FFV13yP5ti(>^V)1I~v-HERAi=<_{tm~y zzA<(DdxZ)#l)|Zo5Q;HV8xpB5Fy(!e)>msE%Q8dNVCoM&Bl*f}&j(lQHIGC#htwLV zHdlj>Oc2NjUtC~qVi*;$VD%WcGc2*2ck^;~VhmM1m@tfXNtvR&#KjPLYtk=bSTs#m zLf_Ah5`sF5w!N$_L^JX#Ni#AsmaLFyvE8^=pUHt0>&l-VD$9a?-5p4GC2sIb;{c2Z zdw-@T_#ESYZ?G)TK9`1Ir|_6ig>NB)j1UYEn2%F0ocE}I zaR10kTCL7!2MCHX67GMOMN!shDvU?_md75CBIBz5OP;B;FG~tOn+YWC543PDs{k_o}Xdh)cMuNe-e)Bl^=BZFo!r5 zg~tygI8`TQ{QE$xYUW_v%SCdkrg98Zki&MJ`lH=9S z9-Yp8u399po;8(+X3E_L1syMz{dn7k@gSby$+*;%9Vg2NG(=aS{#e3qvGDy-D}44K zv^J-Kc)3k?vQ${l$%}7>Q+$k&Y5lGi;a+TjOwTDE62TKQiJc!+QIFv_nmj*hWH-0uu5Vgnf?#f6Io4w5r8`QVS9dzj57IH-}$o+E%M$2-TfJsTwKD^JZ{FZHHR z3PCP>1(^%PP~?SstvehXm2PimCSoW=6`e|~r--q{(1{LyF zS@|>j`0Xds5(P~~Jf+T{KWOwt4vh1!+Ql2;#^Z}wGeyPfrUjg43}I1}ZziIBerQIn zk8Frnl5>n-o`_;yo~X;?fMa;#>g}h+t~=PF#j-&zE3+f_Luv4V?kvfEML(al1q<7;Z|h|1x_6T}+pOh1s!j1>LzM1q0{ z_q6*kzW@t1uRRsXD|r4pzlF@12+jFi1p0l=M7B?&t=!3en6TdM)x0i}!FU7BqUJ(> zEK^-K?UWbYC|86H{EfDGqK_PS0IFPN9cZ@xeirvMeYa@yp4e#^*OPg(H7IjNDN_}6 z*K$OD`Xxlm7W~&OEw^3`Ps_}Lb^P*ZxaR2Q(N#+ZwJxSkD8a!pO6>9ktTO_p$mO$X z1E33&(;!=GKwJd$C~}oyG9vu^vA9(W*|#ED-N89k9P;%y<<4i6#gkjrkVQVr-{sjW zvES;mZcsuC#vFW9dqS&eKcg{9+-uBOT~QV8A=|7 z2&r!k@m`CSj9!b}gK?|dpmSDlT>`ON_J#g$%D0@R@Cs>}k@=ynn{hjN63<$0@URZc z$hB_Hr$eB`C^z0UlTu(col>!J>Zzk~!{;1?$#yG4cb?yblFfZ>RP7#d-s_HdWMT_z z+a$3K&AVA6;x=7Wc#9ot9%2ivMItKBe9e(Xb@gDS(09MU9luC8DxTyUc1V>EEp=uX zU>9hpT48)yNO-q}CQByVHvis{znQP5R|k^(NHV6RKvJ4JglEazM~#(B8y%m-exFzp zuf{-l+gFy`NlYyuuWdk+K zn9qVtN#eU}_~q90G`SK1b6I7k6~hrI%)_P4t^p&?AGWjg$dO>&!G&6RI?fAHh(?y6IA&V8z`0!)M?UI~_O z4`qMV`uZh!m#u$mL8MIb4ZK|cK^}kQjv?HM7hwK1AEOm6vtSKUKd~7jTOA^MzF{55 z$UNEK&TCSo*C=e3@OXBJy0KpaLH4&oF<<4ql=28&vL9)0vy~5hV2&1D@`TX>5!%ud zD^*o(`uBYIBrs7pw)>QXK_A4WGbCw&7}ci6-1$fPDRYm;yqJjLXq*AV4Jap8iZ3PK z#x(T!(vF;LN7)YN+u>S(J@kzmGZX!I-#MRkJz`bHDUM3_|AdH}(*z2xttbBQHg21B z)r~#SG&^V&A*_m>4T;g-xsmT zdwHnVk6Nx*>gUJpS`BKYa2VTTQ+@TQwm4w^s00*9M5XlK;ls>fto>*7i}VQ)8># zq=vnzn6W9R8RAn8ah5&iCLAFl1WCS{h6@JXh!mPyiOjaRG&y*R#8aZo6XELR%>AO~ z1J-D7gEnQBbY?;Xq|)JsxIP_1o5biI-U`l9<|C3%kG$iTTia%|Y!h)^gU{xlr}mxb zAA62Zb=&&Zald>ve*crMihU=9RlgGnEvaEvuz&4!!C8g47Btpq&Ect{nXD~0*H*)@ zIh2i6u>123{>Avq+bpFR;e3%L+IA)UJd7z$+ZcvRB}>CB<=qQI*i%U-0AAU@&a$i_ zwX#k@j}&b0Ee>q&IjdeFC8+~MSlGu?gkjaKB_&#r^yo`8U zUo974Cn!JF!RfO@qLb&$w*L8r7NDUZAc})-(Abu`e#ArxHrik#rllmR7URrT)C#=V zdv>|ltPIhzV;+`S%yt^e^O>tw6fk!c4|-mb{xvHyZr(?Cfte;uI^EHS{3Bl`0ZPE#vo@KGeNjv;9aoLH-5!5Rhb5CCTT{$9t>kjy|o(hyARTOBg^ik`}|Qd51D zdd+D?zf-fK%eALQe}}RZnh-aMPP~)Y&|PXl!tL>4E1IUDlL09XQy$fi8&Z-VwOUzu zxb=#~Su3)E$!3LMRAfxtolo=E`43TS98TsEp?$@5LK_&2BYFab#Ao-=l!R)6c^OBE zNn9eaigwK|5Hy)my@;Rl0a2F?+Lud3;wWn~3B@DR-e%HPnb!3si(th{<9VIQgDbp& z56babyejKvfJ!WbTLQO2FKYS|i-wS; zvDdJ9t(}%v_A?85DuF!+TN2Fzx)A0qK7=B^!e}rTz*?-Q<6Nh9wh-uC690`+yC$NR zax;d0Zd1Q~1gpK9@N}V-Kmq<-5}~?|3oh4B4$etmK})M(Azif|EPc z9szLL7X72jjGCeD=nt>V{n?n5MF2L*dMEJwPM+qe|+x^3!uJFc)bPe3kKm8pWyxw{9*&WobP?aESj9X0(viBT$>c4?Jd_9?V$#=&&k-cZLr@#3f zf}>Y~SSk0m7C+Rv(zQX)^u$iT_mIk8IcE!y4o(Z zC(5^O{V;Fj^EjPVb-fl4B`3m^C(#BKt5rgP=Xj#Zalh2T^yU3a?5%D3?NfFW4Dh9< z7I!b^KTU~?Ndjh&5V2k=7{)mKSipBaB|cL(`z;l85n~g3x)A5DdooyWa68VTp=Xzy zd4b{#q8>7FQANK6LkRs7mWVcU@L;|Kb$0^5{8WOW)qfRM-_?Ry^4derhzXZMDtT`t z`AN`JIF`32y7zzy?OpH9kb$wP3&e=}>YTZ~prP#kR9sY00$>bVB|!$fSeHfzSVy8L zCeMNG4CQ7IDmA|O9JsNHQV0t??xsnlyhqP`f|;d+xzT1n*{I-upYY#(zo||bDNE6z zkrWUBZ7zHKbym}}R>`XD(hnZpVSreK6DF@0tazgcGH=B^JXL~g{t7a9yjmZ+YvNfL z+q()wZT-j|KrKc{P#W?lF$X)>(m#0(Wy5T!AGVqOm&G3=2lhJaZ@E5=ak29n*W$s+9qGv>QApFP3^iTg&-KUaqZAv zu^$udep(rPGJ8#`tj9k-J~Z3C@d`P<(UCj;L;s}|4>ZMHL4J|UNnVv%_+Eu6Tec3N z$Or{_?kT9K+r@K>1K=3`M|3!u(x{<#Bb|M==x)eigQ{VO?{*xpow%(y_1Xwt{Gyqm zi>)VHX%K8V1;oRo(AwU|Zp*-D1Y@F}3!eP`os$lM7={^Bw>Y$wqB`R7*^WIT1Jd|u zh2|++_PH3%REt`8kAi=7m#K!Ge5b>Z8N14sg0wP8bwzM;I38}dW&#w;CI#&S@@GT8YJu8v4Y39wx+_)4M2D7^HcTqNBVqA*kV(07eG67yFf+`5b#Q&>@coSNFoC`I2rVe+ zSp8Nofy}RGgefM-&epZg*~rP9bSrBeEQP$cO(6T@@YW#-#rv3qU9XEim3zA#-NZ-% z9zt8N2n$ru`qpSSi!<2ZWzygY8gl6yoJAiB+Jeuv7+~Kt*sJjR-3T=2`ilbsly53E zVN)oWHL;&2*YsT~C4Pug8eIf_fi4V^qd6&^`rr{_yzBN(~izng(EgFVp*vx*FheXpH= zn>m#v_y)!7z}gg^qikokqeiP^9Tdhmje@Lf(=M+5=Tt55Y?z`mkY&U4*;NAGbS=t{ zA}s@>l2kq-hj5g8%);>2VO39W{r=^6v5R1cBPnLU=E-ez&hDq?@>j)uLp<`VsGmZ7 zq&OS0G|PckBCMpd1ynRcl%I>{uzpqDy$jThcF ztEi-w@a#5MR&ZRf(BWN>-!ilov_Ge5zKyWKU8RYkbO7{aUGD7I?-Vk-Sd3?fA1~-K zscu5C_sBttXpV`#oX+3((mpoMAWb_v-gzPR?;l$A?}rE!7HQ!!Wi^sg;@ZO0mbe#F z+Pl{!0q}o)k6Z+!-jQ1IBw`z5&1PEtLK8V=I5H8O@|cO%VNghf<;FkQ_Ka)T z&}owKs41X7@l%y?kXn%2j>-AGb#p#bu@1^)-u8b9rI}T=TaILYE8^o{koAq{`Jheya1gWE1es1jCi-Jqd{S{ zjYcP&YV*ng?NZGn8dl8N9zt?YSBKMEbS0JXh5GHgm5 zp*`TO=VMr?F9<(q=~)K4y|#XO{5Q(n&KI=a6X)7wm;N5x$0}9$A*UB`RA3EA!}gWQ z`r-h+#A%<)c=L&9)n|jZXjKZLwU9{#YEWN~pPw{^7|tNBxx#9zcCV6ewK-MB{H|}a z0({uxV`^l^NLE~>E6U^G7geHK=udl(6S+;D6;Z4s@q(apH3bln@$*9zoz|I$)S97m zUBjj}ld=rvD)mQMJHsc>x&nqzSoEYJu63mKlTT#*N<}G~M9}v>qhwf-9T#nT|0(|r^+mW8D*lJ(8%6`N%$lhyuLB9m^LTQ??fAO8bPSww&h zpoF}4cEkM$;FN#ur6QFp>_ib$;$Hg(nm|whGO|s{7ruuow~g+W64upwOGt-}NF7b?h`3bj3HhL;H{H_cXDFV7%7pYASN*m`Ut{+y{+U>Mx~#%1E;GN4fV zl^D-W?0+Rh;9l_KxZ>uK<3yY+@nIDfE>~LgBGsD!UsEcG{04X}c2SWQBCC&nk?3*i zX2OS`X7e_jD-tJYp^9D3rfsd!a;S3ys#Z()iBW8wwvtr8$9}enjvC7+@;%-6zC}l| z=Nv4yV;2ubR+R=NrPhrWN{rikx->{^d8q_D*N~)S9gJy_ns026Wg>AU!Cxi3Qk@Hh z5hdqx7hmjoH%Ed!(pbY+)?e)ji+UR|aU)bkk6tPQSY{FKfN1!mC&rM5vP7l5?t0O^ z-h*o)qAs#8^E3ksBg+)pK-rnl6cXw&S{K}#aG>q;Oi?SNX+`-{%06+Vzo$INN(GwF zLWd6#l?5X$-$;wS<%!}gg(~bm$~QZ*hRC@K_efSUY$j#|;nw<~gVo-YwCt+iFGin{ zP3xW6&FY;CbKt6j0F+0(Ig`61hCE5;=KvI1#|bLCe$Ino4n7BTW2EC_wgNUa)!$H7tat#XARk^5QXVw>5!B1u)mQmJdX}vuAQ`WJ*Bi1O6 zJVE&tgLQmow_kUYEYO=(H3&nK5wE0KO>xjepBMu7!9CsVEKgtuwUECB<;Qaq_39U0 z&xR2ut6uz$p2^J0tW}rD;uv~*Qq=-Cb4|#QSYh2EXqFzrEbz}Yfy<~PMAuF)X z_xpvj@izsvn;`7vEy|BIoNW1NyG^7`sic-U!CuR~-n(@1cJ1FB(_w ztZ?C8?{WB+a#+f`3!XP{Bo%ur{fFic8|3;PHU}kp{@zl8BkmM+XxC97g+2oOr`Crd z>cycdRR*-+&Bf{>lPWhjzDI8oWSSmP5fkx_d9{x8L&??a%jemY_E{|Ij#)o5{o2WK zx^>nf_iKC?%VxR7{F{XDV!IesQ8|jT%H4i+pNeNxSkmEl*L*DKMRc_2*j=5NJ!C-GA5@dJu==F}1QX{faKu8BoO*r&b+yC1v zT?|vhQe1jgkEV<%a%RPNLSxH!p?Dx;Xy0Gube(jha|%3cJh{sk@y zBM*7|B`&UR>9;0#_W>S5tes;t_CA040p==V7RzSE>Xe*B7BRWqrcp*vH_D4eseH~?R zBe7>aRj&4$v}sS11!X%}%?umnx~@)yL#8+>1i5Sdu0aEQ+3u2!{ep#Jwc4+D4289hg#)niszhJQr zZ7t#E;tqaMmmJW!oxLNKM9%Kkah1JI6VLtbU5Z>v?0Bxm{+iCQ484TUzo(q6$!U0* zVTtA8Pc?LDPc>AfiStnIfnD6w$&Yt`tY4TjGqzabjvMrFE@X#)UFALc^7x}OQw|V~ z|102|^9ld?_q>M8PDZFm?1%|=$er!Rm&@l7=~4dni~0TpB}R|1nqB2rx8?iJnl#Ka z^I0b>)@v_-B)<=RT*$)~H-UcV)|VNR%T*fjn7A0@FD{~-9s0N~^O!48oiBgr?JV+s zpm}v79ntK1Id%Ij4yX-yE+66*yLW*kM%6gV5WXw$evnFKew67d7d$_886@$DacKILfxB)jQ*f@2ySd zl&v~ipg9EnK6)0`y@tdzm`$~qz$y)bPUp_~w_4S3OmTGLD`p`fR)3fC$N9@8ts@Bc7fO5i}h z31oN9VR@Q_;0ZIPM2NVmR^8bW-KTl)MmSBrdTPSUd%TZYM~Kv9i``XiZrRoqUI@DK zYxM-lU{-DU=?EYVImO`1s114m5{t3W-gG;ME6MYzkz;Ne_GtI>V)2x1Q zdEk6q?$rbfFV->Qr3ac*^U2`}!#5RJ(JTA-8Kn$Sl7dy62U&aBc31rM=?3o5UI}J@ z!+`|063t-0f#6a!>YB8$&g_9*nET2iBx-wGKmUv`Iyy>wX1(%{ zupbI+eaVJrZH?*MVqZltDBfAyOgU4={+Vv@($(eLBn=@<@6vBl4aO6r*LZiL zEcb_69{9l3n0)-WXs2fCMhA6)>=$4+cYF?~&%mtYru6lbuJN+uGSMbO8LIH0>}Rvn zQ@Vuof#j6!vT*nvH1{1wn)@Jae>XM;Bi{W!?gMUnXM-``PYs$B;=f^}r7}YBP~*#A zqFVki*`h({dNL}BvQaShY3QBv7S3Uj8S@dbrDWyA9eDArOJN5bsk|b#q8+HqwY^<| z0m#noDVr_5!Th#UrB?2C%aZkDBC@eauS=Xwz{YJ&`fz6bC|Bv4(7Z8|P(^ZL!3Erj7L-GT8RlwMH+UR>blZG#N>cR@*yU|wlO72O7I!ZEH&pkp2 zlO@vHwTZv{LMqRS3^rb7F7UNpRY%kcXMYV6LU)g){YCIA*0Q(*H}OmpcD<*OlXml& z^sA>ac5}$AszEsSnmAc34!RLpYQi4y9{6e(PLtN4=I1CMdMlQ3{AftLH-o21U^P0$rvkO8$#LyfaggIrS!-_?^gBtpH4=ct@ad9v8V;BKwfsriyaO(-g_m?!;E z@HGM)>UxFOe6tgYoN#3d@Wi(}vU*=?wOw!e<(MBaKB_x6S3bw6yg@74A%D47ecNIa zX=HFhr2KV_Xs9suc1DfhQEy^l;4NdcQ4b8s!+7$^%uPR=ch{hE(*r%zay(JZ z)kN{@6m67OEWgUDWQ0=_w;~AGUyIzfYj1>ZDbqGrx>!tGjyVp87rEq*rH6QFo8Hyg zb9id6(&X(K4pM@cdDDOtyQiN1`GR)o%*u9_sIo5EF_Lk|0gmV1L8MCcC$60Opo~J5 zs*zj9BeM))SJtQ@i{)vl+MaJ9za;39Qd(0mXkN=cFwPKbZAf%(AE{7-roLQgbVz!qY($cSLq*3lmD0NHA!yc!jLtW3EXLB7ztmFhYXL;_qHt)a|c0r7Rn2kSp zVo6=Gco&>d3~!ZK^(WOk@fbHOZNJpQkmlzTDcC2H<;=(>HWSgcFgonE#)Zp_D#hN0 zJp*#SkLSgva9{{OKU~s0bAl;{_h@G#MR6U;=q$*7O<`TVbDa+5<97HBFHZ4IdgcZ$ zr#^qOsph1Ad@X)D<+>KBjMsdlbnS)MK9Xs`Hd5&AMb{-T!iS7C4iH_I-2}?b(yNOj zt(zKv*ewgHH)Yu3q>qZx?jXPzH!Hato5%QSZSoaNT>!>>6&mjI%9j5vd(}>pLwBH> zC!b@F=}*($&e;ClYa{Yzk0%XM5|YQ;O}y};`?UKs|ihu`z- z;P@gm{C8Xdi8%UIkI{K+o&#y&o)j@+Ir);A{HXd%cDRl5s?NS4Of5~ILgI7pc zE%V~0sF`z66S+!*b0PYx9AFhpYT&q{UdurHFjV_}oU`py>7r%7m&0CXA~n{ur+4TU z_yx2poN`pn-3Fb=u5Q$Aq?-rg3s?DdbeLdIRn8$`#`)r8%+D42MUk z^md2)1$hK`z?#JSgM3tJ-6KdpoyU0Y*~AZZ7g@jyMkfwBeuX}Y#~m$FvYCX(_%_TK z^5_SkYyajT;UQLk5TY^g<ae<3HTCY-*Sg zJeb-VmPVnslIRXjh+gkyc+ZxnXFSq9K4H6@x1UhqvTm#K+pVlUO}fR~k$0Ombtgo>oOmH(C-=q<($CZE?@jcMbWIln7B{}y2YH%c z4e-uj=!Y6tHu~gGa@O5;!LH@je75H>3c`iWQJH}5ymXN(Gss`~V z2bRI+ljvf|Dlz4ivnvdV&XcMT3JQwxpDl!@KYKq8hbJKs@Op%(KoqpMMhbqO@_u)= zRv+ZsL{zB^$gEu+8mBPU9~(MH!$9#J;Hz@I4wb_P{oN(2m!oHTdfhCKkY?@cDka|K z-v$uc0u?y?Y6_SO4pD6p8cpD$XHUaY*uZgNQ9&7}I&45vF)HaAN{DYpYeya9K(p>% z)|M>H)^vRe)Ntq7&F;3cOm9X56GgpU&Ja4Cv zkB411=ftGej)- zdTcEBSB&fNHZF96sE(lPMy;UZE}nHEO#$>ny~Q_>8lHX%E+TLwSmheRdmZ#G0%iS~ zF;WLdU#7sebh>l3kLhz84vwulUC6LvWdM=Qbey9uq5HE(Hb+RbR-;Xd0YNgRmLqC$ zsWK5SoXYZxWi?4sP4I}Tq#IX2>=zot$;Xavw0I100JQ# z*7gd@&6pkbtAYAKqv`kVCmXwpBGBgOT&`~#QI?;7WUw^a_nCw)VJw*8E&<}G$6dfQz`i;_>iu&(gDk7*C@7;zsO zL(>pP$Dt#6*LgTm7Cz~vyq7|r0da0=&n#?l(9?b0rLZg z?ktuDs}~hOjsS=RtdQb|@Ift>O3?*gU!CWKZ(9ir#?T>L!Gyv%Qo_h&59%W7Olgrp zroMdu5-6WKXj)Fa-Bq*q?6`ohWx$BCPkpk#Dyi5Ez;MNu{kluXP?1krCN%+454IPA+%$f(QVCyJge284RIQrnrPyvSb2;(r?#= z8~1wH?~z^TR5=$yXsL*jf7h>(R+r&H=dUz&WKC*c_BDSffrZ)i=cTqasG$eVS#g|I z_0RWd1rpA#;FP8^ECC+>4dxjS z7dU6JJnxmEuuxznj3X|$WsQsTy~(p+_jf=YCH2B_45>os(M=j3Xrs_&(9NmSU>boL*l zG1ETK5v>vQS;*ZMmr*xseY_6)4?L_d_6NZio6DB9dbwUk0NH?0B*1@zpev=oYb*+r`eiy*yiY(X3*xsz}?hiPz&0|CUr9-tk6LSGHo-=2#?v#7Z1<% zP7BiJ(?`TR#sxKuK`=k*pvhld^@IYg$i}e*&diws8`ODKF0+!iduz~MRIY10Qi3J! z3a|eAD7)CXtc)J@aafATYu(i;7U#eYp`{(pD(tnOd<|`heDs@7d@5s_p(W3K4p_Zz z@OM!rg510f)3VV+)Rr3^iJ@2x_YCz!nB^~D`%aYBz-7b97bNbLhZ=TE!tE|INb@aG zuE70lFY;Zn)|=6|ktsvYAz*XeWp_Jrpp)LN6r8h;$=G*S3Ml!a8{cCh;8)gd@WLj& zPq?!F4~x)x_<)I?7k1Ty%i}->ScqS&EPk-hb|KC~--kuL^n6w!JXn+=WHQ%Ush(69 zGp5QPP59VexosaL^oSfUkGET)3}gvR30 z2Lh|3`OE7<+k9WDfW+4F1nTrrR)mTWPX7iUYo;)v^kNc!KoiHwVuF)56B)qz!w2Y5 zGxdkq`Q)H_@Sk9~C9*;LWstc|KL;TV3LLvf+Se(gF6;Vl(aNDiF8_Koeg8zA3!1z0 zyrp6yQTREGuItSz?Xvz-&|3-q`6m*Cm%yh2mGiTY_7ymS%*xIsO&!Hqx#0WL-lt{yIRw-pg0(g9|?y$bwlNM%~`-y`@9cZq|y zX&~G_I*KDswjvG(!GyAJIq*J?bN>&pF>rb_2J*;2q(kBd&eO<8B~bl4=+^bT{@9-v z&ezV_nqIAUgi|2Ua}t0Tg&gMj13%n3qj#Z9YJc^!?t3lSC*)2T@pdk!FT|)o8ffkg z#Q*DyKG+Xh9|%26z6mXCAYL{hNM*iI84_Ze_&?#H$~7Uj&S~;IceS>(wl&LXW2niO zuN1^%KYj!w6MhuD<=mxj`wjt_=0D&$(f=kD&4*NyQ8$UHK&cAER0_Tv&zJjW6Wa8) zFNMZ*up<$8?X6}gTxi;K1+=DSSJYW-0t#q$TMpqTl;rmXqF%V?#Dy9Mbhwd3*7adC zZJ#86>_I60uhN45!%P0lKV50SFI$&>lqCD0i^p}TC{{ZTJ?b__;omNmhvDhIO9%<^ zU84yKgg~QGZQ9|9dZY-SNAr!+EjudgOmUIo1`wUA0ITLT-V;kz?Ggjv`TjG0{ZBCA zBAyRuGQK9AI*=pj2bK@jc2^JAZP|j1r1DNfK!Epxk{pQudMDvcKmoNiTuu1 zDbI}~eWyFj6LZY6qLO%{{p`fxQLcwj1VR`3g6~o^sPZplbKfvBAhGg;iTMiT5_cn| zo4ZL_5+k=F1GK&j%S-^%L_{U*rS~8`QSgM6g9uwu_`A6~_%=6-07S+McYq6_6TuW5 z!Cx!{$PN5I5!DK_69L>LTX99CL|fGA_nrTZ96Uo8=tu17F66(-Wh&VU_ut6D$*m-b z(a4A*%cl(zWW5)ELCy>8eXQ8^lKn6&&~VTQUz7f4kNgiml8^i#T}MI}Hn2uQ2CT5< z0to(Vw)wwS%beMF1R42_9|Q#C{k_TLTf={FxK1`WXsG{&cail>ul@Dgvc*@ZXl&*ZhOwZqD15Sn2?LuHtHQ;lDWi{}fXJKuhYz z!|PC~_y~5Y=~`HSiBlgzQRu^za$K5iKO|;C;QQ+&2%v*le+Y1XRjg7D6hH%k!a3;g z0(>ys&H1_#i`{|G{hr=z_;&()u;Fj_=>C zfYXNn_b4S|XFvh45y+hb|1N;shXDM2iG}{a=Q67|lm3kWVI;sHWf$=1*8Jhr0C{`C zzeXGeU)T?-cOc0=EDP|LnUX$c{O$Ju_W(~H0zVBFt<(WA_vXwA0F=KwKM2r7&ASLy zljzKWpkMhR*Rc}5$3j5O9W%u#XA`9QAaB@tp_|gO|Md}|aJ8SR;PO&=-`MLOz7hiDfsy^+ zEdRf_xc`~dF$#_T9#jBYKrI8b-dIbVfNGhubujr$Uw?8&P+u&-6sgppYJvOcN zKkR}3$@TtGDeH$7vU)-Ry54GiY{B&$l>g1@Gku`kBS1Lahd-9pUL`&5@5^rs)R2#D zt#~%nJn#o0U|XJ3|5ZcEnZO^U>;+RQ3k?*6%b=h3-zcccr^{h82xz-oK{@+j|He*? z$bq)|3&kFk>?1XQ`}WUq^DobIjRTFnXPm$vdlm?&Fdvh55dXXSp+3|P<*%mP0{ndC zFI3hPf0b4m2YV2N$I<_q9=ae#KISJdk?ev=ew@$Z;qWa`vPCIs!r-;bXs!099S`(-$~NhDi}1`&VC00T~q7_NRppC$ym;fs;m!C0TcHKX1-OHLJ?iGqQ5SHC zsgXZkzoZLLB-#1fl%4E3pt>konI>%8fqyjV*t1h?B=-0zDt~qQV)<;2dVlc(h4flY zdww*By}qs^JKAJRg!%q*4%6}SM*22)LJxD|)mbClZKjV&ak5Zywhf#0*X*9n^I@&{ zXw4CyKT?-S3bjn`3=GVR*HZZ1Ns|6!3_c9JWkh;<>E%^(zm;x{8M|MljrQq=H`u7k zMNdjKeU59mK%?t!_*cRqmdBL8tV+ViV3d(8s^mjIpk>f?eV>^nLq}Gj)%11K>D;;+ zq8p*}8%-|f@Zv!|VfR2fF}haW-Bs??Lz%zZMA3R?*t+6Y=eAo^ajhxxUh@s zy+VV7iIN6nRPXP0xglJd?}8t^2m$u?E%t=F!cK2o1@Pn^pD!sBkY<^Z^a~h=a~Q9# zeCd(CebGyzb$4_TQ=go5NMd-LaY)jY&K5(?UMqa6=iO8&6}!FPcdUuwz(h8QPFsYi z_}`w{`Z!FqxmPf#|!5otA91aI~u z6A-=H@5BTC5WzUSRij1gd=;SU=PRPw#+`bvM8n9z$MZ(Fg?o-VhO{BDsWY4XKJ0uv z$N9?sM7UD#4U^IOmZPTjl@Kj_D3g$u5hHds&N8cYXXotd@Hb*IF9N}L2H$kykPdG` zww01k1X(_i!mTQ{1zK&~V9PK3;!8e9_oVR{R)aAWiHn!5&URmoEhl~XYc_G%@a;%S zCSQx6CyQ_`Bcgpeufa52UDii+6mSPmT7W`-3uHV07g=Le;-#fOFLoQ5hq* z2YHJhz@7zLo0uNGR*Ffu=fT&O_KxE1&J$tmPNg$f*!SLuSSSHuj?M=Yy>@s-BdS!Y zs)*f4k3szOlMWkjFAnei9cS`*Ug{j-BT8kHqc*v5;Kg3(z3Ho_F4g@p;aRRs zo0+*epx;@k2P=Djz!wpV6hiGJw{`~}AT)Vq`f&<<%h_O)0u9#p{gX*)hqs`5x?~8URQ?=Tk901vdqL|gkdupT^swcG&cRk|OatRYYp6xQUJpQ}hw1c_h zN3v3;OD*7;I9xC}z7qqlOmX|ty(PC=EV*o*j8~r$L~JNq+Au5Is*nqd2E^KfcQOjO zILljKqZT@VS`KURHVEw)RG;CFNg_exhZC4 zHY<3}Xvhn1jB;4BCSW_K0U62WqFrG&>%4w;x2*W5yM^{i^)OyK7!Zx6;T=e9f~NAl z)Qd0vS+4=U8|YML4Ql2jS7-O~Z(k`}0+bn#LRv*O);?xwlsOf(jV|I=1Mz!90h8a6 zKMsO0mXPQtvcM5HN1yp*QvxyHuVC(9V}d3g>Z%*J|2%Z|4IR7%MC!f=l&aoLXA5fM zy4s`>69VppYz&Q!^1M9hd1gqkK4qPEr6_ne9Q-*d+<`$v?2Bk3?UVHcuQzXAN1X)o z3!~-q!rgY>jhUW>-O$Oqng}^32QNy52w?E{VIb5)B@cdDZoU!hN3FtF_n%Ox%?}GQ zSnZguOKnTnf&3zY#r7$!|L{O#+I`G2;!wdsmBqwTHw2S!36{PFUfmPFuk`|6Je%s= zZ6z!xB%SON3^-~P|L?epWB$wlL{I&*cq4m#Sg1N>Z#&F)_$rgqeCYYjFGjopg4)oM zV15J`mKy=KnB475ar#%bbsaRPweS5T`J^gil3ad!t6%fF>%JRl9uqyUanw;HB}dZ@ z9|-_j=5MNc+r9m-pBeQ z%amRt(Cf8%Mn;?$22S%vM4bCe8vdjoSq-Pk!=%20it)MJM?gjPiLkPv@k>1n-oy9| zyWaQ-axVF9F1b(fUUbKf`6a(TN30{+4q#5oe9i)l4D;bW47r_>Vn0`j1Gc#iHgf4c z4*-W5Jof`9c-&b$p2zp5c;_Q`yxlRp1QN#M71Zq=9m!eqgba}ahj`G3Q+$HFu@nNm z^TfOJY(akaJSH|;z}r(P)N${}111DJ%VtAVpp8fiP(O(~gY~p6+hN^!sLqw2guZ*a zV?6-xM+VyO7Puk^ZWwrui!DwSAO3`$RhB6OahM120;>(j7!MxT!|$>^D$=QsO&NXX z7ty(J{lF1c*IyOyck?!Mv+8u2395VuLsH;vx_4r0jH~=t;ZLr#B_4lJb&Nv_aXkz+ z<*eb&q&!_8T$BIkZww8uG6YxG^Z;Y{(SsELow(!Xuf_CZy%2{%c%ZV&MA(0{6;=6n zn?ETcj~OPkQ1z>K`1G)4_V{?$%Hxv+Pl7*&;2NX8kar!`AQ6?Enpx=15W+H|X6*HO zVjM5{NG&b8f(kIP8s&RLJ)-Lp;(#8C(K!U&+SZSbhD93mBDb%J)hJ$g6o*XHb&QRz zjb2?fFke>K0DNPfcJ%Ct^7fHn5QTiU-1wSl%!9hU2?9Vg+~t=cASloEhb}X0t_`xh zp<0#Z=l3qJMUz|SHzcqNUZqs0N)Ebkj+w+uTbCYQQ6<(gWC5NiGv%n*Ejl*}mI`Fw zE9eO1Ov<*y*YAwxUzG^`MhLSBlG9o6ScuKT+3e~d+NlwO$mrp&UB-v|LkM0 zI@tYqu-=VBB6+{xGM(>Q$T&F-huYb{|7IbI;6cCBG%uhJ0G@>)N^m~rjxchq@Q7p6SJ7MQwIRFX)Q{nCC9ESA5COu_NI+O|poo7%lRX&{g>1TL ztJ8Sm#NL5!ucnP>RPtb)&t|DLo-;XNh%X+oHl1!resqJSWm*cxh}p(baB z{^x3sY#MN5b(+%Gr%0%W^G8d}@=F!WQ`NTt?z$Keu$D;^(+Af+>C@=&s)B73AMlD1 zuO8C-yU zv)H`L{hhe#Rnbgr*HkTXvJQ;WQ(XCHo2&k+;&>GN!)$niMa+w>KL`}xr7`5-)A%sA zjwk1+fh}-YS4`Ns>H@ai^|^9D@1|~X)#Cdn@ArHf61yE(mvH^#B>p0*huER%4NJmy zu$*RK3u(x&8RSn)a|Xpxb`Axcng*)V3EB#}+i`;kVRQ-Hg!kfBOsK01$csC>KeOSE z0it6cqKfBw&m;w09G)$(E)6NIn?<$1xSTFsa5(ORF?n-G^S(swERf$Zob5@>0e6~# zTFA7z_24GSXNpdQby)WNxtUFvfr}Me*v*2^FMa4m7oKn{jYxe~jZY-WT9rK4wv-bY zt?;FH*oqyK{hK+HTzlV9!2DQg@Y2|=x*Np6ESgET6<=v`-4xr3{XB4}|$@M}iUiS(P|&ZSQzf}>I~ z#6`K={b4r;)2jQE5R8BVG1vw_@d!19t+CiO!7XZ*Q;vx9FWwTTKFa1n-i2T< z6%+F52xh}V{n2Sw#d6(2;ta0H3Q0WMIc2AzO{9_vicp(ncU= z1-i9ShYRgAEsnOQ@uoIMI_8>kikj`F9v&BmVKD-0xwMTabJ5l;8%)usJ&WHoZ8zJV zd%unK^LY}8q=~ulq3sf$$zuSgo&+lN8WSm5e2>QRVDklB87gTvLK4YO7)a$MbS94Q z=lx;05qX(tzm%`6xMW{7th2Ye_)^Tm-$2mw?fLX%&!4S32j)W7p|F94{Ck;qaFppz zVg2IIas$=6tMTr2tDlo+{Gy&98n56-_6?6Kd` zytr^Nm8sxraUfQjgIX0w#-pla986)t>^k{PX-bui(6f zYGiil7xz$QB+Rnin&O6;byc>pW6cRV9@lpm+`5cYb{lY=i#^q66Q3jXcux<4j`eFi z*?swgb}(eteENK8WAy8l65V9oQ@R(5Hd|H@(I?ztpu!(Bow-))kHfn zOiAfc?l+En<=o>ooZ?V7YptbUQ)+j2bmmgc@GL~*p4jt0fl5#eAkmfXk3bL-xFk~* zwV|vC(^9rXL*Ax@@?*D`yBPE}psVeygx!so9K{do%90l^K-Y+MiUh`5{) zWm3(^;oPRm&>Hne0oDk3{KmEwj9dy1-VoOFtDwnWIx(u*G& zR-PX^vV*a(mM!q4>q+87y*wf^r*DVXs+@H>nOn;A#^}UdidIUl31y5#;Efi#INwYF zhjj==AJ=wcwLJD?E*Q>ddFq?(bY(X4+S}@O(}5lU%}NR8^ccEeyfpTiN?; zs|gK2S%S!wtBle7t@J&`H&(If#QU0lS_@f3B+m|b58~Q0;2qGCQd9c#e$#VD{o1+&?Dzt%$*wEgw(ZHTX|iqGwkJ2)Zt|5U+qP}Hp5}9ZpFiRJ;oPrv z?7h}{KRo0b>Zt2HzWYVE9=!o`+)MADmrrhU9oh&uHH$Bru`gyzZhS@PSop8eS0 z)uJ;K<(LcS)Jk68|G4Ky5q=}%DZK3P7b)p;gYDq6BnFBYfj7pVQ|0K5U&a;l7~ZdGen<(jX|M* zt;)dlzoxzS(>fIF34t&_ZDY7T1^{;G=u^&-MIKMCTaV--psR~d1_DW8Z5za35FKNii_y6d%Dj}*fw5FC;Lrb z=G_DitL(i|EgE)rE)F_;dWerkL#a!qv5)Z-@p9!62)Fx+eFN?kI;P zS~*Mynle_G2ou{rqp{%{bEF4u#K&a%fVw9aptn!h%H=uHF4NprkgA;=Xsl#lZOl1@ z6y$UR@qdJNEv|<%VuO*D!Z&x_VW7p<^1Ic!t@C|8uueE8R|$NOeUF^Ttp0~UF}YOe zDFvy0sLPyRGV=y4Mmo2|RX~+6!#)z2J?Khb2Nt0Xam+m#O=r}?v4+M=QZ`O&pK;;D zAvax5Ri}?Ktosxt=XaW2X2kt5fU}_t8gBl84faaj$KDF_Rq9s%O(l5g%#fcSgnO_% ztX?IrH6O~ki#NO`0LMRgGRg3{4YR~Cfv3>3XB#o1^~Wa;g)$dtCc5%a(yrF2nOk8Dk7-gX*avEW-4*2Jwv^S+lIy#x&mCn94_3Kp*iXe%4z zfaFR1w0lpP!Ae@6{*lenEizxqr{9xv2?;wjHG=OooC4`_B2ukoiDe}RIz-$;))mdQ zdC?mIvaFF*lF}$FK28kGV>6psp319+`b%8dlV;=Kr3splWO(u|9O`zV3$ z`3KpOS`bBwZIxKC4Dpkuj%+QfSs*pxgH_){ zjqHQa#?;$4GouAwckDi+g=fdy@(N~hJP`s+`#TOV*p!}xHcke!s3qG zNL0oUW@Y+v(}b9*@Gj%M@s`>FPHLCU5st;tx^hg`GcwS2=R;IRi#Y8$@s;5O#oB1= zJkUmZcp(hM$=_Ni6R)@1tTHzT*}w>Kxg)iMxi6OJM&JGPo%nWC30=+e(+GPh1mi|? zI$~7r*!9zX%m28g{%>M7lIf!WBBQ|KvmCMbQ_A$%Fl6w_2EvGj?%K|cN0ekWQ{cwy zNSsia(J&N%y|#nYt)8%8dOW9WRYPXs8bUc$B?^WHJsxxoHFJqFl$f$vKGy?(R3RTF z#JDM$nL)xOR(H4}T{S?HNir(){Qq+Ou3E_*!;g zAnDO5fU@fd#rK27SealTf5BY0KERf)H~kEzW*>V84>5|KRjmY47*}KSChvb}BSs_= zGD!C15CTyJ@y3Mqazvzru6kINE{TAWD%ySzbbTeXgLzY)=!EpE0lr9XeNC5s{j&;g zY9n+sRDbLChq2z81*voT1-sr|L2@de2t)ZEK{M7)>;(0ELBAs|#GVJJHLpgw`b;^q zVS9zj1lSFy-Zw~!Qnqri9f!CK-eg@Lku6iss>bWxYV?FtM(bK)=Z7(L{?4uDqbR6e z22`l!e^pb^br3jkqva$By3^~I7}@Jd&m?^~nJ~p<1idR0pg~NJ?g{jFbJY|(s&!HA zwqpNM@E0?IxKq(q^`|IsR7 zfQ1?n;FJCP#R?Np=bC#Mw5z=V=NQupha~~_MX=&uK-fQM5wzHMg+I>S%w4!6bN1z1 zsD-99>oHQ*?-t@e2yU?^k1DGPHH3dm-iG~U#viRjYdwkO30XA7I`r6@15%yM?wJ`r zmh^nY-IL{>+stTd1pBfk4$3@Up~P0x;Dv0DLbleB2>NT3-E9*3G+~nZ+p}}W4^D}P z2cXpM1Bycog@bFss_02S07XimK8kFHpN_$eBF`~`?x!Wh4F8-`SJ?LfMQqa6a?GMv z5t;LcEYV;GxMl-UHJb?xVw-+A+b(6g6nQctZ|z-h@MgT@eDWt`Z?@XF?kSJwC7$T) zJe*ve!{#jumOJq!_>St?md+nPK!jO!d1<9E0YhD%@UZ1xV;VH?nf)^$^@3k3fD~$; zd-QBSn*zh3#{aDzvADvu#nO`way&Dhlf=`SV%qpjXBd!vcdKFGHCT2~J%GL2CXr`~ z^FwWCne{ZFRc{XaOqZt`C6EQphz*LK(amJlcD z6!}G4wxCR~7dyDvH_n_aVd#sxJZIm#$@NH9xE=Y(Uzb8XXb+fzJW0>bL~G~rZ6US| z6(P{vzMEPq!@{G9S#;U_EP@05+6DUTJ$>Dz0y0E%u}E9NBJ<(YahyP%7|YZQ#i6## z5bw{2X9NVpOUD<_apx!a_Du1;zLlX5Cwm1*YDMEzf4G{5kyyO%0kd@BY2vDY(oYd zSo(iOAGuJG`ZA&I)=>*uRpiyg>-?`Cek)(y1aE{|cwbl}Hd((T`@NqMo*$;xtOdNT zaP~&tdtDugxNVQj^Q~yu6KJX@X*SXUtHem%7w^B%EDGlFRy|#RMK=u$+)hj?B?tb3 zAT5c(I0)~3I^Qet-JSuwT5nJQnb@P>^9`BG>;No}cHMX@$V9kzE=J@nDNBvI&E8}Ay|2K{G#aN6CkD!^$L@9Ngz!+CJ#@sZIFUhI2T`G#KO(aNNQ(4#1dlW-%a<(*9TNwDz5RH?OC$^nj$7y@Eu?})Ltj-Mt1ixXLQeA4mk1{ z$g;(d@q43W4>fIc9|jHVfZhKqFd;b0HmiI{@;zcpq9js&tK)F@&njdeX__Gd|0lTSb3ATgX0Q6^G$`5f z+OE1SvC-RWIrBzJ`%gQDnBE`MjyUJev?~!zm1>x5M5)Zr&1^w}5J?LNFC;3cZ#FxC z;6_qR`bKkuFM!{Z`vni)pDHvYLT>EdR8S*d8)w& z?>niQ$w26pfLs{pL!|gTx9agJnTuo!S*$?CewQBtEQpo8T26^7K@8`nX-te0c>+Zw z$Aq<@pXiRaWZLIR%UdTCM-1~80?r(eAdiB9wSMocppuzG{Pnm|2lRustG4J-OkLvv-USpFVni{Gpu)tdjN08Fjr;$teTx@w1an%gFdeuRZ0O6>8Y z`D-zat;F5SV60ryPcpn;PY>BQ;t+?o%LazW*roagF`Xqanb;ocNlVP^Ks2n9t4UQ{ z&JY=W)S36$C{J>p&A?4h@(Fh@Qe(GluDs4mJeYmO7N;9o);>KNZudP_Core68Ycwg z55s}F2hVh{+yA*7a?6qbVMpF&w60O}UCZ-ajXvZ-1J4Jgr;m+}_(%ukiFSPT-dJe> z0>ALL;58{?r5X2GB9#(Q6pvmFz1x0O;SbeWRO0Vrqlvda*JS*>n`r-5GjtGX7v zU#E@nUJL!Cc4FpHp?Ca_Fczy+;v)Gf8jzNG+SjupQ|qQnjE(;KC{o!c%K)Q%&7Lsd zUX4k!?NKo^CVrjsF26(ZhJw71qX7iJT7UcAgvB*jt8#xu^nNgA855B0`^5nxOY0u|5v2wBo$z_|x@2}p@COp9Ze9OFdL5|49*(G&Mx0%Lt^n$Nd&8{I zqZ@rEbR#}=6JFM!5pIh=@_^jFN%g9JyINu;{!Ggf(a3;jw77wUh#8f@MytBxVxsVS zG~1mrr)N6+rYlF&!|I0iqMmptMbK|s?c`IpS#eEve7TBZPRrpi#_kUX{otpV5oozN1WidaEDK*pEwYAb^bMS*1470vhrwv`$c}>G! z#lFDWmk1|WKs*uM6xr!@NidXgRN>hysTg3u{CBSx!XXcTOtZQ3=*1HW->gnWo zB71)st@(6Q4+>K%L@yC1wFLWoYC(Jgoj0&o4|lg~UoPNZW7AP!p|Qz&_^mOw&2@j+ zUvg+(%U{IB2kf;n3>>t^@@!vak+>3jOSb{g7PTh8!7+EWNr;l^m9W9VWn|iU1aaK3 zyvxuGF6-3CRw>rnuwpU~l`*HJ58q+GnYE$9i@Vi&dVp5xrl|f-4b7C<{qr+CQ#=!q z5F^IG;Dv^h-okbOgHE&UE22Yqb3_%Eg)RXq3knI_p*8JTA5Mi73}_j>;;1WV7KPFVA54ZmHHfcZw3!G z__z}$clIGe&&QXByqdwX+vf<+&%{|9zG}bQ=km5;bxaktp3WfO(78@J<(NAyXA~=` z{bKq(;2R(%sI#eui_KI}v0{JSqGEJJnUZLWoVb!s&=%>M8+sd_9L&Wa5uaUxMN6x5sXM6d6&b^u5RYT1g;70X_oi!K7H}HgG~a-->||s!$T^9a7BrOsd(+p`^t@t}1xf ze}4UPzJOBzpswo~n_qRGTF39C_8|PZzLo2u4au)3oKU6LQ}j~nT70t@l244g0hLI6 z9Y#NIlN^Ti;=7vW&*JtyS0HqLX6-!_#s%`e*45mQD>Mgv9aqaW5lQ0QIudGVz$XZj& zz90&cR~`vv`0ERox0m`Ez^UiABH+fT(;CT&=3dO)&ky*FSY(IgO3d;}?hDq|?)3hu z$ht|whyG4QMfa2N#Km&)M%JCK!1yO2w?`9VPux3zn1bg;jrh21ZGU=s_5 z%q=HqU+Yp>2(|f=ynmcls18okxQ_1!Pn%QJn9jp7J&jTnEltt56q0V%Ls8c@GRIxGv;C#o|XLc)4Id(hPDH47mi+Vbzop$zXg;iCPc>Hb{piOks^iT>=xq~bx@GY`h^cE6uL z>|gMRU??L*p!RyBmXi;*ULb<~hm_oJ9TM|?y%gbEn*`Y- zW8+gUMaSMVrKk?MF~8SR#iJ=HZW1@;cGfX>HUPD8!ioa-*dPnalZwg0#Kgjw_n+Ov z{KrGlKPWiG>t0EEBkju-56#vB?4wn?6l$w{k$_!D`8ht|2y&1IzZ%qJjLrmMzMfdd zAl+p|MNJPk0SZi>7YzGBXZgeEv{i9C>nb=cpO7M5#$%}n(;k$ZGc->%N%=Z(3^VVd z)Cl`D+&XWh^VHGQ+eRgZX?rh&*)l9osbpU)bi(sO=?rF!idd!NI`t$sEQf;KWOoG1 z%!pK~iQo%I$w@}SeI*Ulq`JQ4tGoiHGvC2O|IDC^v(v6m@@p`%Cb2Hwd5c^(pUkqj zm{G2rn3zL_e^%6ha^Zi4hi_v$hF{eo~dx39q`YmM&>r6X5#X1@mxw)gS!&_^`lDEG(qx?PVB;cj@?X}g8@9I zv9hzO0AG~>K*(8;D$<$fZT8UBjNUup;UOYJnOIvR|32J(=hW!9r@R~7+6ev1F@SuqdL1d8VPmZ?Zyh(x00{VkBUVH=_q zoz2%Y1}zx`oU2}0@5NX(gthqcGtOO?ip9yneZRB~AwUYwwbenQ`&?w^dweu_z%BZk z0*;i^iEC1dc1fw-OBOZ5{lWvPBfPas?WP4u+F{e?3-~$HmUDo_qS3SgYbti^LSv)s zIcqnQwADlFkb3SITXTA&cM}tH7T~b;{Y9IV5oC*11RKz5`X*M9L%vdK6iD)0u5N#D21t0qm!956Y zbc`W(ce7}xVw^FZ@WASMncOC+%xptMwcdl;cJp-Cr>mSKoKFDoe!Y%A6ME2=)H7MS z3e}mhrDlhj7#H44YoutEsVG8^11fc#-5I;N6yx`V9}#NdQV^NNWgo+wK!RvEl8n0f znR5)%9SPIFywlP3Ysko@95^{E+P6b$Bd<#+vYiilTqVhQ zxW_|;@1n4hxMxhlL(QKUH|JgQ8nt$uxxWI~t{crO1*=lfaF$02SS4w3dyvOk-qjnQ zgB?e*)f5zWNK*$4%D}l}x=U$_nDH|JWeR`Jk z$P295lGm}&x3Qipa`K-lC}Ytsh2hR<>xQGhdW6MFI1GoZ%0aYjQlVDdj{O<(@BuST zV4!bfUK4Xat3T*nK6TK^CS?9km!yTDweTd!J57_OUPdNoE`H4k+g@#_>D1A%6r;OL zv?6o0opfND1j*7Z2lAL=^%~V}?avY~1EI64h+=#Js1^bySDp*E`| z=??E{^5}z|^(_0rbK$+0(K~Ib-gi5|4rq@>iLBo)156PJ8^rB{(Z~@*WIW|0@~VmN ze+jIgexQp6z{w4ffF#T^U#wSgw!tI;UqBmN=*$JE&9IuRsLxaRZxH8D7pl;|aj8s? zg`Sq6ErK|e&z5v@ybL-*12uQAC)xsaPQ$;&Iq1E|>4gT$_$md;7*gn?{Do53z26Un zW$PN18y0GmZ20Co8NN#;L3b7qd$*Ueaz6Ki_Q(e#&>g0eFJInn8W zL}oeX>t}eXbtQJhofiob@4U)eha@2QGB^yf@#Sd2#aNcf%*dnZ_F^{71Ftyz*v!=l zTN>(ar7u)qqf^3)n_qG_jdT=OUm2X?+9o%~ZUBHW2F^3#$n!>^0Mj8mmh`?gw=r9* z9w*N5@f!Ac1Ibzmb-;8atfN}!@>E$Am?m8C^(4?yN5QCkvz=x`HMrtsbZpAlK5<#P z%5ABL7?)2hcTG?^S6fbW=5|=3J1vt57p!M4d1NOtJt9++cY!+s{sL6sH&;e8O?mqGU39+5aI+OY#qCUxaf=z8apahD{y2m)u)p9Mu#Bp7uA8123UWVl881$yJwZG_+(pl7|9m zBq}7l+%&3F{w}wTO&Uhu>dP{0)(l{gwhT6Hbu4@GAsPw1C-y#R3q*@4?V9)eU2zm|k^we9SJ z$1dV&Xm^Z4uy_Xh&vM0sLKU;UvEavz5r848p}Vj5LzW!DyfQU4$POdxFOdHP5%Rr# zSLw^sN5go?>CLq3Mpu{yKcU^9*P zayap?d2prgRuolMOw7h0W*9KA2=%uX-HCC8`~esNF@W0>I`_qj5*->$OmRfWGnC5k1suhl_PyTrpuJvg zj?z~4jAa_foMQKaWU2uj?0y_(>aJ6^*n8afn~E%^R)gsnO%qddrowd4hjNvug*J3R zlJVY`qv%9Ty>7?JLZz+a+iO63!QU@ewR7VSoS~0iMOo4lEIDLCEGw?>A@QV-?C9K|3w(z=6V#BCGNbR)1N{=q_GrL z980zL-!I*aOaQ1yb1S(%bRT((p0&*U_Ng8R|PUgHO}X#TpIYd6{vNDST`UE{PMpHze+nl`IU zxW;f%ce|E9IWPeWtWNqM(w+%i?|qB`$3r30X-L5a2A_mmfQ*h1sN&*7B-1 ztruR$)q6_zJfCfZ-9|cVmjaG-Djbq2worjqAl(t&3M>~99N-PM5upP)6rsgBou0_| zD*}N>C9<6p2xZ!SR1wH(^1BmTM}Jl_%I|YG4L%U~rv3<{x@##IJ`%h9?g$D{cH}1Y zUhqSE1wS$N72{>&DCfn<#2Sxc|Eb3py_YpCDa9QPRxGWUjPOmY zj-(WN8^H~}V`K3%zg+lt$4@F1iwi%GOyfrt#N^=~Z4C(Ku%2h||Ndjxf23ui83ma^ z=ha9-8_WO=p`(rbCS^=ks){p`|#2{L&%7i`;2vl;w>^-CC9$f#2jm@WjV9wUhxCiB>vBsI+HaEe`0RDR3O9 zr6Zt$bg*~OC&xcT<7bE8YovLR8eB&~1GyeA+8^z=N9o&#G*7x0X=RP$Tnfj9q%Cdy+MVq+EZAj_DZ*W&d%@O~5pD%5V@pyR7! zN?}d+2^aryOt3x8mb#WRaWy$5l>cfC4_WGs>!^mB65dU`*gcG1=wHpfQpZ;AQBab` ztR0*!2R?aNMf0ML35>4Oi2`Q(0TSKTcz6m7{%BqM(pCLYc>6>|#w845OIsfK-}l*{ z;b4boA;RY;czq~)>xjSA8@ngZnpqQbGVQu>GA)u4r5)l1zV7aF8%>>aq-w;B{QoY1 z_po2IO43#`QWx}2{B;SNR83exIaV%HEXc4-8zQhw^31TVN@uX10zg`=Q9bPpV^}nP zX*e|cezmEaic1qe$f14gmINwJdx{O5Ya&ow3)*Wbw!Ad(X)o)R%r%99+T%C`+_v3) z0Z*cr;XNF4|6%s{T3rcYNn6rqppdvY58|0j{|eG!x~ue&QSP3DQ`|2kfmW$T-|2c! z!F58gyH}A_euGji?_5T(3RQ-EwM(a2E-riK0SsnX{3aaPN$9j3PE8gOeH%vuQD#62 zcT9jHyo8pA%Hg><3^b+%rFdm4st7oOd~R;{iRmz`FgY&14z|NrUwlmY9NY4cJmIRu z!(g~X@3bOHkaoY2BCl#N*r*{gWuP=1rqb=EP>_v-yI3Df?ze{Cjm#+f)y-8LXf;T? z^VV&CLEI9SnLn>xr}4nV=#i1pY5HHpaqmIXsXnK^1x)ADhz}L=NH^4_7x@T4*=C?w zls)`~%_=hycDPQR^N(Buc&evM3Dj3;2~|YKWh%RbwWXoTvFhB0?>DgwgSU3YQxZn9 zFkgxt&*7h9r$X$_)RC@@@%-`U%*@(nk*2O=^ca3WV+LD5+6C4?AG9BOnok&OCc!^< z5M>%%ka;lQBDT!%mjqM)70xnY)yXJX5f(H#Q}v-PUw{$Rk3W_gEWVBJ^37}%1}F53 zJwHE+>bYFJ89Q~`kcS7Y4Fhc^nrocC;wug%(P-aRphMYUQEYV;Lw?VU2`e+d{TJz0 zbb&#$0&=>_#ro;j&WZ1=o)dorC+lEo;CLBLo;KsMVmwOOh9we>h#ku7Z&hHPiNB?l z*okdIWzfLIHRMOl%u+)JmJ=n#`R69nZPXBlSzB9_Vb#L4eX#?A*A4%#j@t(0Ck297G8Gk7UklvwlHU_}lds!1wVH$Nf)E(OMxv}0J#!dPWxon(AzbuLS*p|1N+(AD6AQX!8-%|f`iT^O8hNGr z)Zl7Ys}BP_BBRM0%i@dy@bLLt!KI_4pVoCe4>~i%MvJv}i~5 z)fwgS6k!=mE7wzMn#TBIKYFE@a1W`{Oo|Cfa$=Lhyd__nic}5_?K-%j8L~1&LAt}7 zlWj3}(B;}W%IB#+ab7e6IU;2bTt^cLL((6W6RRd-w zCaJ0Jo!^6v*<8=SYEOu2f84Y=B8X$k^`wrbp^|6t_QNE2VvbEl?zbIzP;H{IA?bEd z!df-b{m5xl9?E}WLOQuN_#QYMlHju_q-XuI%?>`pPuVqgf3ho8i$w()bd&EjS8tI^ zy%0bxAM76b8Moq%|3Nfnbig*$e4LK{{ri?#udw6+`rVo*MpynP=Z8{Fmb3k z0bIjpF{%v9C_ zMz)h{SqWM&jlv8OCam-(fuu$YqZ#HUdNj?iq_oY=Z!5BNSgWK>o~@>>sac z!Jn`8peU{t3(zf9CoY(u(r==X3kos44IAfpC{fQ(B-q1@;k26Y;6V<$=w)LL$lB*y ziZr?FGWm@J1o9V6`OfW+Z>vl6ibqF{5k~W&#K?W zrAuOb4~3hSW7~R6a({U*ocR3r_wPQF{wEMlP~s6D(L_C7D|xjq@Thp)U~6l#RRZCn z5X07FK$3F)kxf*b3E&2E`k?OiiFV8=#(;1-_^wt_!HfDG&dD46TPk`ly#rH-uswXE zSi#ShCkDaqf+|@YKmio772VKp<9=NqL?hA;h6yP6?ox1>4W5196Er@#V&!-QNP_&DZOwtF;?8ux4;B89q?tenV{45)cFDYlwe!y~Awc^TfW~ zpVEYwn(2unp#WN4EW-*ved7k`MTXdCJY@-12^+0WyUzUh?r1xouQ9a?bFCVT8JRt)TDUV@AQ(lMOgB zxLK`RWQ?@Sg(0df@fGB3g z3^$`GZLQAae*}~dmg+!^L#V#0zkA{OUcVaXZ;_=K5E@biu_c?AU8T9A8;1nR3BY)v zdOQyRrqN8b#iyaZttPqilAxef;Fok&;7Hh9P@}^!zgAFNDKQ6kT0D5~;OnSNc&##T z8*a@;P-Npy%mLwR<5t^|Wt1(3b>~Sc(&wCs>SBKGbO{;7A(>MqV#tPyohiETTbc4Wg4#;;%2_EKr@% zB5NzCBf}5IfLK>1Xx!Ph7c$)9|8xrjLM&Pudn{-@L2e`+24&~*?MyAzgH79~dx^_5 zVTp`hc4Up5z=24ZxKYnLJ$`F0Za;ZYv$cMT_To_b7yf5-Xc%j?DBXMcxz-{qgoVLw z28iW%LFF+?s2L>X^b6gno{5@|Enw}bp_yJZVdd~wQ6k2W5fKBNlF7rJ@JoI81bWs= zpqj2$3HUdRdvT?y<3U@s`LTARQ!4#yj3VPvdVHqWcO~1~k-%u}W`i~D`w;CtvQF_f zb9vf%;@n4~$lris-#_oKUMG}kciF8n!kH{dR+1JkMDu03*8`gr3Su=2b(UVpQclH& z$@D~R1kNukzf47q=9x#NARoHqvZ8cP)QYQq@u9F&B$g8Q@k*1N3r!v3zzplW>=qmd^_BaLl;sR@v0Mpg7~_Cg;g_}Yr< zr7wkliAHO>9-@EDsj(6X7QP}S&D=rv4{ahC6m2M~g!qLvDFzDGDQ2X9!V8PSX)zgE z4wkFUB1?36i7rQNcEaWT5UC~*i2UO=e@XU9L};C{r1{gxPe)ELG|$QL?3{M=YcHM} z@;`^(v0EwtFp%g^)Q6&=V4dMc*@Dn@&G;2k=exzxGEzYY^!c}-KnI5db~f`ypk!S+ zsL??J3;}(zo)G6>6o=2oUWLtf%&@o|VH-z{xgr}l=i{Y??EBRY!QJ{oyXmHLeDn1N z1_IwL0`Gx{VQ)u>lcNkq8)v-m?P7LH+InRhK#zOZ(EjVI@zVszYG^6;HP~EV^Y$Iy8L*+m z@>y8eaQSo=t=dRnpofTdjr&B6QnoIzrlUlIS+x@4siFjp>6#&e*-fdoDH9#0zO)Y# zNVh>r*hcjgb%Z*g2q%EGdGh;Yt<_!BPPq4K7&O@4vq&)(fTHX503UncN2V{!-7GyA zHSah~{TCz!^&cd~E)4HNrMDVGi&>nL=g0D|faUde3~szW`WCD(qur60oe_-X^&;kt zdhzXPb(iHF5J!&zAY2n|-+EU<&g@~H69~wVB#D~pbk3+4?Vtm$h1k2z45O14tdZZo znJA?0Iab-FyW$K~!R8gzH0ZH<3)3(Yp>}JYlX=7~YC84pv{Nucu0>gN8&yMno%*)C z5R*<+S_HoX$x-dGNvoi&T9$WVqiWp5fI>Y%gN_l1aA+Ye2>EVPrhlvsu4xRUy5d~P zU?`%f`frM%uC&;saomKY@(2Nt)+x|G@u62-Ba2_qw}RmJ0W>Cqsv=Hj zNvPWvY}LY%Cfng2D{)7B*iL}oV$iS*k-5CVT2k<8?A!eKFzbhyd~QG^tK6_cr7zVT zeyS?OguFN${##|z6f=pB;IHRu*PV}W66t(j=P)tif;%uSr#JCGhwZ#&ki@Ax*0V8Y zf6;sm_370yNA0YzANI(+0xyA-%Lr!Mofl}w1sU;>C#W9%M)*&;V3MG*KgXBE?Jbfz z!A5%W{b}^2Gb$Ne1Z-!S$0C0jxy-ju?uLMq_qLNV*L*vpSH=*bcuSvP&544bv7VLRzFu0w5-7XQ+vcTr!8`lQ@7{v3*bt(S8V=zdSeR}>2O8S&&5@y z`)IWnLG5>4++TN(ypo3;S)DxqOs&hM*MdVVk4g*oyeg8jk#N)dTbR|Lqvx3{CPX#D z6A)ISEtFE#KIOU*?or7MimMr)>kHe|0Ht{n@ymt*ho^KrgsNb;I zbgSGZP0s5>r<#Mx-N#7Hwy*<=xWF=;nk1L{M-H0ya7QF2l!U(4NZeA=8`#N|>YhXo zZ=TuGdgKi)torR`)}Mv4n1RCTro62YzoUEb3@FH@QxvFu3L-^~@a6U{>K}b7?Bb$SX7duKMO@R~jAi~X8889t5q3A; zixUU4zPXGOff##A5t7g*pz7$}&3bK)i2*cAri5Dz3vk>)qY+312Yts(pP&kVo+3R&#Htd*z6#H`1vC@41v$V1oyDy>OkNKlS`;ezY9?a^)Xx zv>Y_?fi&MQy);b~8mQPwjQ2d}WzC#Wk=~-iVue85Xi8tp5C1O9_&7>D}2S%C~Hj>Ij~uXzJKUBha!j>viH+P?}6W6QRd>d zCnn;OD$9f=d_f6aXMB&P+Fk>skMBAK_f=t&P=vLm+NZMFmHP}%?H8OLWh=V*0w)g0 zt{ky1{BAE0!#t_Y8^Tmv{7f$jL^K7u|F!&<#)20nf07?(hrqu^eLC=d|Gn&rn8xxI zT9cT2lX2Z`JFdG)7*h)`S%+G0T!(5BU8WA3HcG!9%6*ou5aXFl+;YmKjhRrH$Tk8_=-BfQM4_%BjzF9$V zInbaIxSZsw(!*t}uRw2}^GDC!%?`DBqua^j74~t4r`N;l-k;>lw51%)2|(+jF)hEn zJexyu{Q5H$uT2!vWGT*!%&^_!z}Q<+eF7=Bom~r$$ z8QqYuRGQL%FKog|*A}&TiQGYhP9?fI<-w*A+nZUEi|Rik`@*7px7YLLvtz5nMC^;w zWYVMzNMU`n6MH+;P;Zl%eJcu`jtyQMjFaeMI!4kzc8oxh>)hSf0Wp@QK&JTdXm!CR zRBKVx62<1lN!Ne&#J=?Udn5GD zPWt;!DsK{@W#@*kIqC|22PvT_?xk}y3E^`F%qS5N&_usd*jswF4vjG`CoCvbrW=dX zjsmiQJNVC_k&RlklctRA_>#oq9bcua^gP1GRLs~{d?@|k-cfA9#qmS;}$$dnYW%cJ45TN(%P68Q>wmz!@Fl$PM($-oHz%c zww}c|ZRGTt-DDqQTGK)KK25Vni8=Rn0b8aOEMMU77g~qmjYfb$@4>3(b{K>fCpUP3 z3PT1scsFge6OXC5f-_K|H*B)+jyd*rVyG7A(22;z$qokt-Qei~S*6<)AK>~`5Y*uK z24i4nup#A$&8mWh$i8{-z<%_=b~AGE@IFBNqEyfLSTCvD5wAMNi$f0<$h&1gSdR?<#j_zc;1)O9xhx@CGH|<409m{;+zgy9VgSM|@~U|6-9FT$+q zyzuW!bMGIII$Z+uSYn~X$)UM0iw?EsUO%Hr3;6{o^IJk%pp`#a`HqBnwz5a&ctgt zB9j|pMO(XpMqyO;+M*u%PuWKo{rwBVa9}!6z3)y-+)w83PQ-Qt!-twt6;AqWI3b@I zQL4YXM#B}|Lb`)a8qDn9AfTE;1dE4vip9l68ginwr>>Xk`_6omXXOtd592UAzM)6F)p}|~JbyQXzj%axlRLYue;{$#$62x2;?e{C-piDBvHRd#EGu18 z^Z7B!FZHr0&>==k!)8AwDq2>KarZ$r^t z0@GQEqBi6>$*+%b#x(Kjqy};xO$dd1+ra5Ok8l!B@g2s?v}s}YauJ_<$8NhcjtjT& zu400taeB6JKnyWHBLG9R=W{dxxqKT>1r4o${fv%8@o*}AEtxj zh4_U^DK)z>;{RWKSRUk01UZuxuBh`hTqj2tZ=~R-Qf>B?6;mOcO!K8k3E!ShE%uBL zcOFb*ke{bIfa2Amv1Z0QOT(M?G>J{R3DaHSH!tfefDhef{U`s8d`a&s3zRwwv)-0^ zOoO6t!!DB6R}^lN_eX-4TJ7JgwiaQ3l{6|9)yUVmobo3Jo)UDqnyG zs=6~kER2tYmc%jMI38tbyFI;{>k(>_U|trtUtJe4mZKcFC>ba7&)Vv@nj*UcQczk(w7OwNv4P(AL%81Ve#bd%<4w*c<*Rcs}Qu zY6U*lm0j)-T;*Sq=}Bk-D^8xhJpe0e(ad{IBcY;-1|!Q{mBVl$M6i8 z;ad~WS9i9Kw<|Kj){L2?yEpV~77_TR^a!Icx&rsti@V3bcs)#nW3N+|b;6J4CQ%_L zt@fbEUMoiKPn4D&G{=egsJUiS@-tj!ZlBOj`+n~ciH-p_ftN`eujPHW+(jr@Tb&wV z)Kt))wJtc|;KLp0>!g?ry8n-|w+@OcSQdwaJ1o991PBCocV7q=+=IIWcX!v|7J>wK zch}$&+#$HT{5DtK`|91BRDD(Fk5gN{b9&lmrl%X6<1(9c8udo7P@dcHyy@rQX#NeL zoeymdqgp9qaGXB^i7j(bhK51!I!9cxz3&#ZJb+vCK8ABgkvq|h=F7&*c$?HM*|YC= zDx=zTHxm4ze`!=5<)=0ZD@F0MGE=|1Uq_-h)D`jG1DZpW=+9Y|I8$2mH4Anq+@||6 zCX77tH+19glx<)>Vm%L%&C6`s%9hk|c&6!u=;T8#{%?*31El{x_?kj454eFxj3&VN zb7`BwyAQo*R3x2+mJOSH&v!RBH{FyNNEb+mipk!6^HdEI;z8nrWHLs?Flsi7gSb9k zSKiCQya>It*FrpVUe81faRhmM!&}0hi$jj z@GjBU0`{&KU48InEm#3?u|FZIPm2)3xbNs!0?F(~dzgMU_>7LwW;%F?w$E@5UuRpaZi%-fr~Vt_Hbykr)|g1)1tB@(T%@ zFvNAOB{*Z=v7ZS?6huvMP`|O(g3F2R4XoYj*f?2N9UL3$`N0o2IHqW=UA3DlJ5sP6 z?5jA=w!FCX{_Dq2(JC~<|*Tg@QK1p_+p*q*2zQ*$m|SDmZeo)h-rNt?L$sC zf{|5fP{=?9^UT&7H#mC;I$Qmh*?mipRUW4T>D71sQW|XFEA05^&u%pOWnEgLEW4+`NUt>qC`EMzpDnpC8qzo zQ%BDB@3O7jv~n5}kh6+H)a3EfnQdGUWVEbEpb|le4#^YNUTO7P6~vpeCBH(8*&sM; zW&m318VJ}Zbt?R2ahQAZ{OGGl>HF%=BUu_}<#b~;upU$<=n0tN~P#D5GB+*bvtIY~r6MZ&C2m+=;_n8|; zUnwKA`*gK^ewFmgrOc-1@p6+n#|4fOr~9YF&xNW-()tS7eklwz zi-0Fg>`M%^oYVI?OQ#g?o_uarp#{QebY6Y%;f0M5FpMzCiaz)@1%DQ@ec|fF{%pk1 zXv{!5r~*rf0|%|`Pge~u4J|CB(a+R~G`c>FE!iu`WFfI(--Y)@n35D)**_5aEm`2x zs*=vEQ*m?iG>9iQu<%JrO)ZmWO(-Gf^Am=USVpfA+6&VI8#D6?~Zy*riA+VOen?Yg$XwvK)578(| zv7~t(Upm?d=+imG=hNNSeFRrRSxpf?yT79ChidL?=W*Z{ZNu&?v@^uh-9kOOzmu;o z1}}-@)@c!9-~9JS85#T??4G1LcR-HsIz&DO2_hwF5yR<1J&|p`oM`JdOQV=4?Baar zdquXpulvZ0_ZDR35>b_Ts%ZIrd*0ptE^4{XnLK18ABEtmI7P3C=I_N)m_CH+o!$43FqDDtY5{m#&)+~-s@H&H@ zK_VA;*JEu;LDKzg5-m^G1P6nXZ!-TaMF#rhMxb_KD?;?}iy!{pMD@tH-K(#b-4JEn zc?>_?+1~3txQDF+{Cur#Om0u`x-@9e)0`jlg|M5oYb69|UZ@&qyfS~m|KAz>oAE+M zziH7Qmj6}A7g7X^G;5DW`a*=MnOvl&-S%c9qQh4BM;K<2i@A>WWHGkAgZz4C!&=X5 z8G~raZq#SFZd@Y4&lr+EqoOU@uaIp%(*&|j-}# zt{HlcFMvM2v``GJ8q#9Q=14_$65cKAR96a3{gl<_|)(?4i!Ob#BhdQ6J_1P$CUfv<@W z|BqP23f27vUX(T8Ap(F~hVXJ8;r$u-4MO8@tT8tp5r6-yd!pLluUPZ^##(Y1QS>*x z!jWSV{$K+Bf%VSsAuq;2b;synta!e^ROP_{u%fU0`zaH zjRs26go2d;5pu)=^kz51Wb0{HjYcrdU9cUh8y0@$SPwN|SA1(;%8+!xXx7n`^Q@Y?`G)6!D@k9z;- ziytBxm?9#T)gKwb(5*#hf1%zR1ZBD3L+-UIkLW526@8f=7LNNfYm}A1z~HH=_UnUz zBWt<^{sJ89%L!)ssFX=Z_-WvH-ac=fPsoiS(Q*8#eZ|3 zCPeJdtWmK34P39Jbq_1?Tdoj1{sJrnnG0q*ym;_Ho?Ise;2lRM`gOAg+23R;R>%>7 z8e-%7X<;|jD}B3-xiZZy_6SZ=nmVZNEtnOWvQ%)Qe?f2TH+s96+Nr^&2=GW1I@+nt zeeHi!>o*9lfKTw`Xw-bl%wS`xjI1{ISINW{2HQSy1>?*tFtn?|hVb81wJ7oTki+KR zvogQ>D)8?m<-Pq+8T9(OBC%mV|zFAU233ot2!JXn1As?I)%|F*&C${JLE zBkTW(27*HVyEFf%{^rDEfQJN874SGmK#DkJs6>bTS=e%#e*^27)?-A2fhjYvoc{vs z3oQc1766HiCYcS^*?m^ysJ}|4;%|yzY0MG)b~peG;{Os){hNBEe$p~vp#03VgL!5| z_*q*y?zewtjo-m<;OopKX-*bI_#_+Y1+qT_Ltwyzz|c}`eCXC-qJNc5y#AZ23iW~Q zGPqE)Mh!SffW`&1|JA@qe*h0nnU!%c#}3BYT7sMS&xrUP{04T)T9M}90Ap<>vOx4_ zU@+FAU}zv0AGXeKtVa^+{))BuZ$8F3YBU6Zv4%r!C;K1D{^yIIv@Ce&{H&~lS#B`a zmIB=Pe@4Xb;5YDf)|xas4;X6;9_iUse0I;Gjms{oSG~}(#zx@Y%hGFv#rho!pJN?GwLX~{n8Su4ee+0Obc3LuU4`uE_B+kE}f%rRs^oOW%|LuU% z#Z`WmsMK79!r`z|zRjTLF>TxVb1)q%1oNAx!k;v(nZS0q>jTXtI(EI38;O`9Z{xk8 z_GBHeC%dIY9#0RF-$vwD|M;Xonj9f1@H|AmZiJcr_MX;Xv9xWo^N#8uS5-ravWTAt1?*p^w+c-Vm7 z=$?Q5PT?|Q5t&hgijoG|)QBK2$P1*;!^}VYst=qLil{!@>>X0i)*~4lk$8NmVQO;m zcQc2W`{503djx?`iq;hNd{&6ihEAZ^H(R0LeIraL5NC->2kS(@ArW!cKWm$VI~j^< zr5!q;w_+LDc)LFYjGZ}pi|;i{aeZ74g#e(5+*t<}L@IUx@Fc84*n7B(Hkm6jBS*)6FU`F#mlHjR_x zhSBOaZ<0}EK$ma~%3@^Q;DGH*6}#w)U3R9!{op;b+Ddg}ZEhlpkEM>fm{sP#^lw;b z{c82Ym5v?8PkZaa!>!8ARaFyun1zceZ>vG>kO$cV{e@0dl>=LSM0n#yJXg?9u%o9Y zXh3pOVS^5|8R{h~H1 zC3z=dWur(H+W{Bw;ze4Yj_rbf%cDyWZ`MxRNpMSPxXW>Ay$Nkpdg3xUCi)`_?W}Oe zvTCal*NZvrx&zbYRvPCLuc26+6qlAHHPmUYwfWbBtn(G#%>stP$qZFiQPv5p%}- zyceuhCih6LE+Ldwi|hlG&RyX8eBvR&H_V0=OqjH|biK{|!-Q5gW4!v(2Ta;J<2;`_ zvVOke5T`cb74~{&@ENcKJ3EJ2iNjTSe!%Czh3^_^UjJEmD((oWx9G*?;}Nt_cTP#? z;lyk2k5B(g!w2zjS8gEf19Npq=7`8P=W#F$+L}iAY2?M#2(i>$^sKARqzPd?TB-;0 z{&E9a@aigSk-UbMZug9_7V>##bLmoS9B^?~#*8-zC8A*DYk){bTP^u6A3w5eX~f+> z!y1Jk1gv#ir1oy$eYDJ#I_QgzLgbw~3C5C+Kd+5NDQI~1`*2X;tll4bN zddpm&rw+o*@GtXJ$po-4f%fFBt~AucVh4;LE9VG57a~}mEectxJoN=^v_{r`LX&XD z3mwC;O!hr1fQMHqmGp;)UlJ2KyIXP9ZFYYlc;7px!dhkaTm_$I6LemqR5En11Rty= z8*Y{L2uFvDMo`yx4Gf04b-d`<=uq($Vs3Xg#W!iE%9huILI-5gwfQR4;+ps18YD-~ zB~M^7ekg{A_=U{Dm>IdF_51>n^~vj8DpL@Jh!6QLYiH-v`34H`@lK04*yb6LqUt9omAxW$=IG{2$dc%0OuiWQ_1!AT)C9{Eu8o_Jbq4;%oYE@44%3@ z%E+{qzQ*dyi6LO-5eD8BPtTF9PEs)YE1zG=msgT^k^7iNqo@w%_wv|8UBl)zm2y$+ zqq~-Ik*21|h&Z6Y<6+6dpZ-RTJYr@VbRsh9nO)88tdOhN~4<0sk9>rz+EXJU$f zICQQK=tOj{aNuN_I45Q8>UOwHrPbA}6m{Qe8b&dje0Co$<8j|+#gAO!1R3VNuE3&a z9e1;1n=2icIkN#Ztw8_#0tHe-A8_J?X#Yux4+4vc9#I_!If28yGAF1 z40B1o+4J;q?6B(hYW(NGPEoR)HCAmOsU`vL1wwdnsQf6SBQBU zutY2;Ds50J*g7?EE57YIjYF1jau5PxaeWg*O)cO)V;Gkt)5jm&RqTfkBgn@HNb+r0K{H5ZS^`le$ zK0C7fwM%$ZUtzuBX27qP+2^XL$)+0DYFBu6iKAkPOs20s_*(s){oguHcT$dz)pHxnIz^f(n-BULd^1TNX(_*2Bau}ASi&4TTIOs|x7_FM^4=vo?SYxb$iNk4=2>iFEhkVe)hcES-FZ80v5F7)SmY%qo9`Y({^sZXR^qEM8Z++q*dHYYAcY zd`|5oCCv4OlpLv2(k5Mfgk~$i^2xY5t`hY(RUgAZo|bP0${YHzqw0XWr?Y5BFDBH~?IviM8sb2j%XGmiMb= z0&Po&%vO+!_q80d5-ISBCm&LEu0!$B)AB$T9D%wORn{blk;+Ts2DmVG8u{ zm7Sod+d;Sdm%l*6a$x5Sa8L;cld!1qTB}teKMs`Dgns zvo3HeU-{GOh~RE~_DEs~ZCu^_uS)VvJB~^MMZef7+y>c&#@*m^SVmZuOH)rSq-VF` zw!GA-m0!-Qs%4>rURCWqVwt_4W9Yf;v;{bsyDM~QwO4YfQ-cotB@)*W(E$IX7$I2WALKG);@CD+QtbH{Bj-8-yUppK^jq+~w=R7NzAfkhZg4KE-z_fdJ_N)$4T1dZL04ry5Hxe6-LA;+Kp|)JEF1e zl^g)6VHKO@j9jw4=f_yUG54X7ux=bDOy?~QeR@+xT*pwyNPn_krpQT{!!TRG;zfhR z=6HIpG5K^$=VS@wHqkKV8efjEHsQfMZ4_KL$mp0anm^>nX5>|w^NOY!txp4$vcEO} zS${jlsQvrz#r_*gV!G!|Hz@0py$U zYE(YWqb0@qbXhIQbm3;1yc10XAZ-VR$oO{kkJi9+MzgszK2w<#^b|eY9oeIt^ zt9zn<#B^*mk91PirTl)H+y(M-sQ`wq{b-s%QT=-oEz04m1T!Nja_5?Xf57k@sw z>Cc`Mh$bj^RL|TfWhHjWr`Q~0e`v%!UtH6Zq{^#sQ|HxsnebS77cM z#D=R;pDS=&W^u0?5Em-cuVHF^0WYj=XUiHab-phA0Ms9rLJ3!R_0#u6lF8*YkGChnyx;ve*cWO*Pa2{-7Z_P?vl5+RWw_e zGw=FA_l4)=WD{e@e^})!R2(9tW-{mK6O``iso=!pMj>-2I$}P6cU8~d&GuxrywW8g zz6i%=Ab@;&Y+)C{l}4vqY@rXz;1^+3D7xki z^zq;>0^UlQ4JsY=ukbwi@}<>x8$|!yG}?+xtcH41cL`Y7#@tF35Ior$@Q4&JHz7Sx9z&|Q$&vJ z4{c30HC2a8)bGd~-Ncz3Ik&ZSmo_k29l++rujYm1o;)O1O33qAs*)t6MB;71{nY2X zGBGZVFe=sJ?=OaqSd9;L6jIYk92yvgBktod7D@t8^}N<7#5X;l_(;^1i*?bE9Zbgq z*d=YJmn-O>(_V6}Mxgr8T2{Lq#$`eYX5}Vu0@CyK0*Ib3aEAqJ@nmpIZ%i03x~$11 z-saa3y}VPnZVGU{y%$%n5k-Q-;ld-Ik2-Ic8zx-4l$8b4|GO3dPQlp9I(NT~=m6>c zwpM(~-KPnQeW7bI7q)X@_ z9pmSsN}2Bp%n>KqL1hQA!$Fnm_GGT3>CQ0T3u)0BG-M8$v`i1@%o46|pIaAQ;3@jU z&%R;s10ZP!l7#OR(XnURN(jPNqV`Ygo)CkD$TqAsB>@N+e zvU#;@s@GKVV`z6Epc;DyNa~J--0V5LNf^$pG>hNcq{n>(oN59 z8!(kjB?v)mD}3u70rP$TJMFer<$!9B%b5ui5<7|(ny;A6Y+59Nq*iUisY9NaYGEh`f>eo>R zJ#ZQJb=o17I2O@Eg?*h{FQVq{Ai;zVnaJVGdLU64`f0$G-71`_%cXv(sW*Q+BR<@t$F+k6ejnZgncUkB^I2ho++YY0aD|O>BchI^#Bt>rB|+DzIVpR zTSfvLbZLRfW0p`sE4*R+Roff9$0}QIN%B=fV^!h?qH7 zu*dm5Scx+#7pH+oDl)d{07r7|mzM^nkKI;@wI~3bwl?HCV%%Ms$lExRNS8ML=f(l! zAEMSa;txi9#jBnOE-d|sE}L}6uJaG&aVh8N(Ywt0yy6SQpRuae%pkj8uE(Vsfwq3s zc7CNS+12Z0l5){X@!O7xsoQMe24@3A(uG`%P=*6-yD2Rx3qUNKyHiCU>}q`PD>h`g zsa6(>(h)v{LxZ6o#IkrHJXa*|X*1|iUp@}Fv{KL(K4-WT5O|glMeZrO53kL z+vi}-FOr45Wx}vz-tdYK`Segd1Aui3b6_m#$gWZK3BkaJ^op^P96z0~?qx$xN_l#) za3rT&Y8zU(-(%x5eHY$bZjAx_}*_axczOCIwilLg1{#;nI-tQIn6hUe9BC!}wS)RgSllNSemL(fH zUl>XBc8+Hs(&YA+b2F8c=R`cej}K0mC!EWysea>cU%bm=inn1*__oxJ8PDe)FX&bq z5yp>m^3M%#WphYM;AB9*YDR$CJHp=9gcnv!3CbLw0e3%x$JN!S#Os22;q@Yi0RXW{ z#2INacD8^&8Jcm(6aRN4_~Ikr`x3?m$GIb)`ENRe0-h77wBN;v)DhUyAs9>`x@7niZp9in_?a$( z;b?Y_bD;zMmkFQNreL@{9>DiFg^AU6-#s3*d0ft($#mVXkx0B$odxen953dX(7U~r z9>yB4eZN^k6UKdawjW;{nhuuHg()6OPx9& zPcco}Bp@`n*F;r+kg%_Y+SBK{aNy^TZUk-fw$#MDi z+I|I^S>P54IByLhOiSK!+;S^Hg*SJ?Pk-K9t!S2hi{QS@_HLs6+iTa^cg%@HP3#LZ zB%J7Ulvhn-AAXmyKrYyNQ+OW85CPD>L5 zVucS$w6{jp&ho4DR#XI5BdoV+zB%}l^)_vfxyCC!gW+vZ<8lBp2|lA?zFTAPjW{N0 zS&8CZEBpDzCg4;9(kA8h{4V=tT$;_R_1D%lbssiEGGbi@+Jq=s)h0BeuqB&VJMLk#tNZNp-V1t@1(JIK0_GJa8hmkn z=)k$nFs52lQ?|(WV)~gTW^Z$2GJ>!B?@mKbMeTVJ^vLxnB|Col3XXXowTvbmuTW$? zP6+Y2Gie{RbYB2GHlf(UvW=d+eZnszKzQtKmf)hr zrr8xhz&5lddRPtH==c@Jg&ye8zT#`lYK%IXP}g$PgCyE~zh(}yn1x{PAb(ihMoXgD z_$1(QN=e81xbtBjd}Ld?T?vY z!uP4(f*2(q01=6^&1*Y(W*Yej+5xz#+Qavz%7%EnPx2+L+S^0XGkdB!)Ga%)>S>zd z!!MbSaA09N-~5S{Zu0wlVJ?{q(hqEx zq=ATK+k`EzI{NLS#`FmKZ12Zxk8sd3sIpseKL=B5;i@8*)465IMV`6gXEm)NHFc=T ziY&T!3?yAP$mmQkn$#k`?@kvcPIs{BnE|#&;vU6#|oWr#4AM-uX z?-m_jbTY4z(k6tX zmui~+ln%T%hUr*DlncjKJY>^Nb%!1wDX-9_TTP$DKkv5<;t6FJMg4XBK#_PAj-E{Z z3-Lb5<%HtKI*|~?5;2l8b(p2zPQqi^4F5)kp0rUc{)K<@Oh}q`Cb=UFyhXJ2ytAe+ zQQ-3=TW(1~jG_kI;#zwJpP8)nGH%{5s6yn@gu@+h)bRmOJzwx{TJK>IQMF*hk?b(% zOT%biSYZ&_yvfhE4#K`DAWfnJs;esJ>D3GHwoawB$b+AtJz@<$JeL8dT(l&6;oD?Y z%?F0n`l`JfKIPV9G>*^A$Gs!-D=k}JieWAq@42Zn2=sAY3E9t&?7P=9nCMtDCD>>g z@2(ebg9?5rMPW69d0kRfaotXmZDjl|upE4S(}S~F$IJ;^o=}lODq7mE$*V3T)m?KM zs;|3wz+HbtFGW-bY?93%JH3o$aUwf8C`uZ70P?#F5(0pWF&;!dc$*|1mAAsSt@Y+O zJgH=^%DVn_(@NS|1kJwdH|d~1 z=^Pb$JH47$Xz1I{qoqPE(#eMoS&F^*9zx##!zjT`q+4rdAai_DmL}ZQnfMLOjiKv~ z-}k#gwso`=I*DVB3rp=KM}5es=1*OO%v0Yl86e20f0k%Sde6bq`8c$YiPXN14X!7> zU_9*UvR=t&1j=fBqMYKpfH=uCz-uEQzyA447(v6qsP8^tG29{3E4ugGdm?fRC2T8` z7;AtR*RtTl6_saF`zoJzb8pbRYIpjxOUJ&5AK2JwSBreQVwGR&x2}oTsR*B_owTKx z=~3OWN@{0d7e!+~0PpgKD$Y$-0%0kG+bBQwf#|MbB*>1n4)05zpOvQa}RS^03j7%PURmHCm84)pNM`gdNPskrI z7V)u48v(*#pQOW;IG(UUg^Wd4hx%HY^7_aPG0^}=c69zG}J!PV{tzCt-q$7lo986H+`^4CfGMQxgS4z7VcPh~i5YIwi5Y~CVUwJRCipp8;Jrn(< zL9c#2Uj=_53Oxo2Jp}4>HhS2H3F*D&U4AK11*oVNdF)ZDJ0pyVpG%{=;@553FE_=H zcl44Dp8w2YQ(%Wr6c!qMUh0_e+QBpQe73cWM(Bv+@M>nSQFQ9Vm~q^Q$h}2#AGnb&hfNr+Q@Epql`mmYOr`SBHmQEP z61E|bB3J6w1sL$!k&q%P*l07#nvfzJ-xIj5#^whftXRFDj$S$WV5L(Iin5}c=!n&> zTF|b`tpX%4hRMtQeAtzgp`n?bQ4d%qI5w%Ik?&?zkzGB2E%S8!5qd^442B>J4=`mVUIxDkdiodjdXcUFyr$b#UQR+jzeL+~{i2>Mc}3HI zAGc{-`IBxL1RXg40Ybmw>b${E0J109I=&T(fwpdB0+~4xP6XJrz=22Y2Kmkr;b6|> zoB?R=hMSYkl=g|K8nIb)^IIGFMh3=*gxV66jPWMM;qFAetG;wqNhe11f<>ufdkiNZCj4mI$U4Y@hl-^OvRP8xY;w^-c;NY3l2h%7OP=u9pfI;X2WFRaeg zyChU+vr97Cc%<2Fsv`u@*AIh7()GIDZ$F&SqzR|4w zLBsvRVkM_TLxcwp*2;L)M>X=kTo5)gjpF@Do5@G*%)A!t=Qakc{o`K)TNK|9AY;B2 z2VuOJrQp33)d4k4Q-W1nD)gbLnKw;5o2Qvp`(9hAc&L3$&oK}Jne2g7b;Kj4Zds)n zYHQT(Jk|Cu+^0#Gk3qM}0S!P+%Au1tF%&JDWxF=a)vMXTVYcC}yJmVZJ0$mr79Mqt z@~t{gWRCuebgL1ykGS~6nC&AJbI;QU?GF9cXi3?f`h-qboJC1FP!Dza8tgjSwU|mF zwBsb$^4?k7Yo?k3A^qRoNZDddn+AO+!{BkJO4}IH#+oal^jQxV zQyqV*jS#*|5|kPwpcDxgb{*a{clHi)FtB2X-(v4OJK`u$cS@$kX=fmHJ);q5#ku7_ z@yUA7bnMy0TOC(=Y)?>rD{&GcS03llIxC>oud&0<_98%$ig1GD;B2tNDsvV`tULO7 zJgi}CN|_GHwXZj4iT~Md={j>#ta@oLdm+M!v2bfeh!{Wu`ymUL;tck>G~;>dv`c7Z z7}M;jO{%>(^)t0;hb#S975_lu!tJ;13bT!rEs^bP<&ObC?SsXk9w)!Ns6fMXV)%h~ zx5-s+m0vi*5T5Z6uEKc#83rY#``R>C_#j9R`{lqoxbMwN={_Yem)}%FzB<)G>Tx#D zNB0%n9}~L4)4@N!v?9GHn2kXd;=IU1ey%*I)s0~7=yGy+)-B!gVFXuAy;IgQhY8%A zUNf8$6$GwgNbjexA0S?x>|aNWSloBFeiUkCT4o$pW#j+{F%fVFuyxSThbGT9;|x!B z+f1l0_r_Q|sU=sX3D{SQXSnlu;%l!WD-S3WwIDu=Fe?8#adDwRO{$XpV{P!U;<5$R zc8Mj7cabGhd;IL~`x^^~Htq7#k21uqTob*(a~5EEyQB8!izS(_D;+6e&MO+{?V`aId_Ct zmsA%ab)j|4rk6N*Nudmh+)p!#sL=N-0z5-~v0dQ_XZV)?an^!RaFbgC<+mb?8)>g= zIcvG|ejNSC1YFmX21bHV;c=~}qh(~9NyA2XIl~WK-@&S%hO1M{vRbkLN{qum|r zrC0Qz;|1?}dAIbUEyDXOhjhFxNVCy`VcR>#Z`+G9uM;6hgxTk7kcsfDNu5g;tdUD5 zf*UcM3K6fAGG1}hP{i=cT(@{t7A5Lhdj5XP(z`qiJIv(J;WnP*xTpS-P^&N?FAs|*wNhH+ zPlDlbn;C~2y20YfFhUv4C+{46oh>8qu?>z@XX}K6AUntlCB_UR* zEo(8C-ee~5e!c8)>Yi1jlyarL-5dI4jGmAf*+N;>=tHELQ~35`Z!=2erBc%`mKEHy zXzTmSJ!(twedIPXQ15^bw%BQ<(qcSwg)wh*axCTHN)us?Qo`9VA&f6Gq9;6N5uaV? zkY`cy=t@J|D+Ez~>sD@hub2X&i=+38q!6G28G-#e1J7bObW%{y;L0P+`UM^OxnxfY z9vFxc6GRygz4S6Wl{=paT+ut>S3{vu>lYhJ%Duo#obwQ_zS-1CbxDHijZh;U@SPz# zaVvIk;gi?#a_X2pbjiN-JiUwL5@)m>hohGaRNj}_2lqN#d7r&`rsejKzqVJ5t(WQS zYM<*`!`%zPO(qhAJtdP=W3ONI9Gfm`WkG46R7}S4efwk_WPd9phcxueWN;)!(crj7 zPiDFM&$~5#!xA~j!H>(<@ltM0=WuW*`zwS>kG-O9xX9?IkRxxIa$We@&Um2~$iHOk z{CwpKa|?M2>6f|7>oU2^`-lKfvV^#N_62N1l})J z`1&9zf80-QBB)}!MKCM&ikeIcIuZ?28_4x8y(X9!p)(e@XTU$SUBHO;eIF0m=r_yrY)Fu`tz!EnbD zBBP`hO1%3d5C_1P*CvUgk*?d7GkFQ6AI=vJjrw!T9xp7vH(qwhzl~!1uAO}oh4~q? z2DOFJrruAY`=k*DdI)zy-d;R~-CPVC(=(R z!zQ!b|K5Y*^Z7D;c^>xV&F;Cpv@1b}X`&j^!>|uOTa?;Qy8%ZihXr@SevcI;vfl}f zX8p7cJ-}a}tTL{dViI?cjfebfB;eS!09(8^vV%y77I*AlVZM7fxtEquME_!Z#sk4 zfZFlE)tbk5#&Rcp!}-77GML>%wKAYsyh3wSNcGs$8pS@QNt7|IXZh`CZpSFmzH|4y z{=qiN)`Qs;;nCi(*eb?K;Sp4gpj}$0uS1X<^-(_fa2>Nq%Uf`W)m-^~Hpm_LPKqm& z4TRb%bATL5uK9dZjz2)5pO1WHj19=x$XPF;1=*So83e_TPCGMFP=6*BQy~3SbDBda zgA}MgE!~ssNUIAx{yA2H#ChrIJZxANdGGz3f~EL-GP7vAKN;|M0(V|XZISWd&RaconZ%7L|ER_ zW^HprGiaeO_k_vaL}`FBa!#E*Qq?Gn!2O;BL)zNC!nsoPGxuYKedY6Qt=qmv+T)}O z%v$_EF8F8*K^g-9SK8337{*>nKHu`f*mA3M^oU>TpGf>18B_pVfHS0#VAeYS}OLhYsnw~iom*uDOZZ25hvsWCPg zsbf6wz5>Qz+k=CEYo9|Xr1fUWX@6mW18XVR*+(UtkuY>EUyF9~QGY8LGnR4y7A6{G z_KmS@LaIw*`(~api2d8Jy+SPs3sS;ZE2MN}-Zt3FgeJ0=->6 z6RqAdSl2`QL?HOa-Y$COBqE(Y?VWV>r26(ED!jAHXd9$s*EG$2&5rDbw8$}P-nI?- z_7hAU1&6&FYC!CKaHfYEDl90z&A}p`T)1B8%FZFFcRoMuaS+@oYrL(KCdT(uB}3

0E@^aTKX3`C z0l&qRivr`g?V52kmoclsLe$q}l3pxe8CJ7r5ihJXk?}l$LI7f~-&Tof_T3WCuvdJL z@>;zY{jah$Zj&$2y#?~f-(l6i-;@j&RN&)0v?FhwXS<^LOjWPC7D5D@33p5jLMiJt z#L#Oyeu$#A%^PL|I5$Y>t_#9suM8s6&Mm!@W1Gt0X%m7~*g3dDY}dfC=}J3Z7PXfR zI|pD?B3W8NJbA64PPpteGSYU~`*>j9@Y6%RwoR2!fF?whZVz~T*Oq&+58g{#g2u0j z@@IlSMFg>vNYjM$&;Fh3kU-Z?@s%0RlrCs5_vr_;g9JdEPvL$5 zfZvL#Y7u?!#Kl!Fjjfk?YG(AXIRCx;L#-3y~u}Ltr}s+_$UE%?cr(>eCdf0 zgeUf81Wys+l+93Q0@EV-Cw%Doon2DY^m!+*?(#)9;&~nr|2dU%xcH^^Bmt_6J3u&U z3S5umWyQdjF9@9n7N19$(%AbLzmGpfyL@Z5cHr|fx&WPeNDQ-7i>;)!O(A7PXD3h5 za}H$t(286Kr%8((fl$7mgi)W7#u==FiA~?5^JYd{WR_PS#QuSa!f49Ap9Xt@>d2;({ zmN~^C%f!79w{P{gXS8PGqggAN$uz^?gNC9!kiMZOR49DeUzN(wWuXn*-agT-t-n>o zibpm}a+SeB(!Izj?5!|Lw$Hs$TnmZF5*Ana6y9N&3q!Gwbt&{M@@s=&Q*n5yv{4oc z2M$VFWXrLj?is)?-2|VW$vWS!yX*L+K3nC?4MafF@CB?h^1*=p3RZl6pF4srHF02m zMYJ=Ce6YIlxaOP7L@Sbx-yqEc&t|RGlJ;bUFF&wud`(A;0e?S+CD2VZ&(5E5*gd*3 z-JlIZKo7(V0*kXg*C*`|w^yoo&!WwDl=Y}REr-$1#iMIpn=wgKdYHcMsi60@RdznW zMU#GLh~|o*Yj)I<8-EuAzYupuY=M9rCz{w9O6{`w(uJ8m`ptyep-bxyw&wc+_tFfS zKq5$Ghjvd`VQP$t4WX1RLw=1?y#(3xpnSp5> zg!_+nR$IDBJWN!z5?~_2O*>u+s4oirU@Y!^i~LP1w{5EUUAnJaBRdCN8TPNF(TAN| zxNI(PMP(h3Dwy@gx6^hB2|JQf&CO8rRWx&CNx~$K6l>CKAueLsccZGVaO`nQ@pdqi zgn)8(;ulw=`>$Tf7>mY>$m<7SRig?j4W*%X(xNl3!Q~qVk8C!WA`?A}ZV$u5L~agv z&|og=-1{~v#+-@LOXXB=nky7Nf8#kaJvp0bF5xRM=Mg+Dp}>A+p%+K#OJ+_NrzPcFA0xF2m=SoZz~GQ+N=LtQOY?sXubr!ZSqj)L?U zA|#w2x9ztnkLus3%20JEzJtBsX?@X<-L3Tky$zMOcLmTtxZ zDr)@TV2_)FoYNOubY~t1`&lz@dckWUb;B{A9bLE*@rCNQS)Nl}3JR{>d92Oitqro_ zKTIZxU20yXt;_T6>D(y`=j0@KK1Z*rwE|eZUm25A!94QKIpz`MWSM_OAq=gP;b-|JNC1{pvG()9TI*mjhUa{rGe5U4Ww>p$1rr*^%C3*847y>vgbmfQN>4_R#r-);$mFymm&(lj%>#EF{Mk)Id*;n1V1p zb&EJL$)HdZ&O(DNB5r5U~PAZ?d5^eEE2R476l&tuId7_2D`_luITB z_}I2`tD*UoR7ldJ91l10sf&0Vu;?~J{QK9~MTYBd0cXUXKTDT6)cdBMtGda$IK^yF zNqDRD7uZcuX>!1X4=cS2#MD?5sYGYTPn-Ev$#5ApD?N|3N7EbWa-Pu#f4hmrux?^aXV5$8>^#Jt)MXjiaH4*=mmPDNTTD5$iYn04~vkVF5-nW(F8aW-LX?D^M8 zUws>1!`rmCAJkYHby52(cS2wI>L@H=mmVb~l-?qEBtY7t;US$xCeb9W14Z2WD-0vl z@Vb>X(;cAljU$NIx~+=WT#>*moLbs?WQ~XWZ-3-8aH6akH4|J?$lVSXo30~1zz&({ z4Ho{t?esIi+D5wh=N5a{d4ilgs&v~jrDvAkQ0M`Babd%MF1r$39rQbF4HudL^j3Pc z{PIABO9Z<8y=2w?m;8J2T`=R+l9`MJZBm=|?5u=;>vF~9W`O{6>{s0+K!FjiHQNUWL0lqvKCeznLT3P4e);T$#%~be5 zGK2pq8pE#1QMqv&QBjWeQRKh5r)yY{!gUCGa$xw6!Va#v7^xddrUhPY&ew+BmTO)S zK1d4Gm$4WY5w=@@Z;PCPtBc+5XJrs@lC;yj!$9cznUDoMzh7J9TG>F6_}_9`Xy|nA z`_3HaK9g|mUfY@DFe~qX&Hs*qDhrI<`XwQP<-D3i@kn6u;$Uk4apJ!5 zhKsn5>(}4be!>{tdcvZ%a+v0P6g3s?q%+09XGvK}e+TSVXTnJs+WGxf<_V(P*?{;# zYH2`7*21utJs;lwchwjMrF~%h{nGijTR@l}w7dJ80ZKQuI_x}Rs|jTySbUoCS`9`J z5(c-Tt?)2hn83Sv`##~{)ui=7j_`bx5f$xGU5C~1ef$vBg{cI)7w5*#HpNw~pBg8l zOPQl*KNi(LYJ2B=aba??{4AKay6~{x3u^I6g|x0_frc(zD@gC;Pzv!;sejwxlVvNX zA?-gJ%75<3K{HrrJTKnr1krXJ2I*?rI=F)x_+f06*68b#;mn(fr)m4m+V_=natp$@ z^x(G_BAq&vH4SO8hSifsEIzlQc{blP{&#R_)@k z%R~59R(vv3jXJZ;B;9Sm*nfAb*8LABVePHmTM6MNMzuQC<~T9(*XsXk>!hu|AQ>91 zgM2h**I3@_d5ow0!cgz2B$vxj~lsBE!db+Jm9 zh{gXqSTWi76~GKvMq6xa*4B8J3HDAF$)nU~fdc4D1Fg9fybTUvtOLq3f)!zw{=?j?`P(A zW;a0E5Cg`mlW^lKxWPm9siPexV{Nu#oJF0iMjU;t>i%rfGMR;t!YLK)C9d-D=tjAi z{Jj%wavNm$h|I^Nx_Hi~%9+7lR_g9@m98F2K-dKKs~@{(c0XKK>PPX`q6nfR&7g$^ z+e4!bN}b%``FGRdZXniRjwhf(HE%A2^?#Pif7j^$>({FrSW>5~ml`?Y{1bS1csxED z=05heJ`VCQZwK5Lo`jgVtgx80u!MxMgt$EJBPAv*CL=E<7OT48@qblt^RRbv4E+CB yIA(YFf~&yzrw31aM|o2R?`KXP?s)p@n!@5DQXIsVlybQL@$PHrs@JJPqy7(&8XQCb literal 0 HcmV?d00001 diff --git a/doc/_static/dataset-diagram-square-logo.tex b/doc/_static/dataset-diagram-square-logo.tex new file mode 100644 index 00000000000..0a784770b50 --- /dev/null +++ b/doc/_static/dataset-diagram-square-logo.tex @@ -0,0 +1,277 @@ +\documentclass[class=minimal,border=0pt,convert={size=600,outext=.png}]{standalone} +% \documentclass[class=minimal,border=0pt]{standalone} +\usepackage[scaled]{helvet} +\renewcommand*\familydefault{\sfdefault} + +% =========================================================================== +% The code below (used to define the \tikzcuboid command) is copied, +% unmodified, from a tex.stackexchange.com answer by the user "Tom Bombadil": +% http://tex.stackexchange.com/a/29882/8335 +% +% It is licensed under the Creative Commons Attribution-ShareAlike 3.0 +% Unported license: http://creativecommons.org/licenses/by-sa/3.0/ +% =========================================================================== + +\usepackage[usenames,dvipsnames]{color} +\usepackage{tikz} +\usepackage{keyval} +\usepackage{ifthen} + +%==================================== +%emphasize vertices --> switch and emph style (e.g. thick,black) +%==================================== +\makeatletter +% Standard Values for Parameters +\newcommand{\tikzcuboid@shiftx}{0} +\newcommand{\tikzcuboid@shifty}{0} +\newcommand{\tikzcuboid@dimx}{3} +\newcommand{\tikzcuboid@dimy}{3} +\newcommand{\tikzcuboid@dimz}{3} +\newcommand{\tikzcuboid@scale}{1} +\newcommand{\tikzcuboid@densityx}{1} +\newcommand{\tikzcuboid@densityy}{1} +\newcommand{\tikzcuboid@densityz}{1} +\newcommand{\tikzcuboid@rotation}{0} +\newcommand{\tikzcuboid@anglex}{0} +\newcommand{\tikzcuboid@angley}{90} +\newcommand{\tikzcuboid@anglez}{225} +\newcommand{\tikzcuboid@scalex}{1} +\newcommand{\tikzcuboid@scaley}{1} +\newcommand{\tikzcuboid@scalez}{sqrt(0.5)} +\newcommand{\tikzcuboid@linefront}{black} +\newcommand{\tikzcuboid@linetop}{black} +\newcommand{\tikzcuboid@lineright}{black} +\newcommand{\tikzcuboid@fillfront}{white} +\newcommand{\tikzcuboid@filltop}{white} +\newcommand{\tikzcuboid@fillright}{white} +\newcommand{\tikzcuboid@shaded}{N} +\newcommand{\tikzcuboid@shadecolor}{black} +\newcommand{\tikzcuboid@shadeperc}{25} +\newcommand{\tikzcuboid@emphedge}{N} +\newcommand{\tikzcuboid@emphstyle}{thick} + +% Definition of Keys +\define@key{tikzcuboid}{shiftx}[\tikzcuboid@shiftx]{\renewcommand{\tikzcuboid@shiftx}{#1}} +\define@key{tikzcuboid}{shifty}[\tikzcuboid@shifty]{\renewcommand{\tikzcuboid@shifty}{#1}} +\define@key{tikzcuboid}{dimx}[\tikzcuboid@dimx]{\renewcommand{\tikzcuboid@dimx}{#1}} +\define@key{tikzcuboid}{dimy}[\tikzcuboid@dimy]{\renewcommand{\tikzcuboid@dimy}{#1}} +\define@key{tikzcuboid}{dimz}[\tikzcuboid@dimz]{\renewcommand{\tikzcuboid@dimz}{#1}} +\define@key{tikzcuboid}{scale}[\tikzcuboid@scale]{\renewcommand{\tikzcuboid@scale}{#1}} +\define@key{tikzcuboid}{densityx}[\tikzcuboid@densityx]{\renewcommand{\tikzcuboid@densityx}{#1}} +\define@key{tikzcuboid}{densityy}[\tikzcuboid@densityy]{\renewcommand{\tikzcuboid@densityy}{#1}} +\define@key{tikzcuboid}{densityz}[\tikzcuboid@densityz]{\renewcommand{\tikzcuboid@densityz}{#1}} +\define@key{tikzcuboid}{rotation}[\tikzcuboid@rotation]{\renewcommand{\tikzcuboid@rotation}{#1}} +\define@key{tikzcuboid}{anglex}[\tikzcuboid@anglex]{\renewcommand{\tikzcuboid@anglex}{#1}} +\define@key{tikzcuboid}{angley}[\tikzcuboid@angley]{\renewcommand{\tikzcuboid@angley}{#1}} +\define@key{tikzcuboid}{anglez}[\tikzcuboid@anglez]{\renewcommand{\tikzcuboid@anglez}{#1}} +\define@key{tikzcuboid}{scalex}[\tikzcuboid@scalex]{\renewcommand{\tikzcuboid@scalex}{#1}} +\define@key{tikzcuboid}{scaley}[\tikzcuboid@scaley]{\renewcommand{\tikzcuboid@scaley}{#1}} +\define@key{tikzcuboid}{scalez}[\tikzcuboid@scalez]{\renewcommand{\tikzcuboid@scalez}{#1}} +\define@key{tikzcuboid}{linefront}[\tikzcuboid@linefront]{\renewcommand{\tikzcuboid@linefront}{#1}} +\define@key{tikzcuboid}{linetop}[\tikzcuboid@linetop]{\renewcommand{\tikzcuboid@linetop}{#1}} +\define@key{tikzcuboid}{lineright}[\tikzcuboid@lineright]{\renewcommand{\tikzcuboid@lineright}{#1}} +\define@key{tikzcuboid}{fillfront}[\tikzcuboid@fillfront]{\renewcommand{\tikzcuboid@fillfront}{#1}} +\define@key{tikzcuboid}{filltop}[\tikzcuboid@filltop]{\renewcommand{\tikzcuboid@filltop}{#1}} +\define@key{tikzcuboid}{fillright}[\tikzcuboid@fillright]{\renewcommand{\tikzcuboid@fillright}{#1}} +\define@key{tikzcuboid}{shaded}[\tikzcuboid@shaded]{\renewcommand{\tikzcuboid@shaded}{#1}} +\define@key{tikzcuboid}{shadecolor}[\tikzcuboid@shadecolor]{\renewcommand{\tikzcuboid@shadecolor}{#1}} +\define@key{tikzcuboid}{shadeperc}[\tikzcuboid@shadeperc]{\renewcommand{\tikzcuboid@shadeperc}{#1}} +\define@key{tikzcuboid}{emphedge}[\tikzcuboid@emphedge]{\renewcommand{\tikzcuboid@emphedge}{#1}} +\define@key{tikzcuboid}{emphstyle}[\tikzcuboid@emphstyle]{\renewcommand{\tikzcuboid@emphstyle}{#1}} +% Commands +\newcommand{\tikzcuboid}[1]{ + \setkeys{tikzcuboid}{#1} % Process Keys passed to command + \pgfmathsetmacro{\vectorxx}{\tikzcuboid@scalex*cos(\tikzcuboid@anglex)} + \pgfmathsetmacro{\vectorxy}{\tikzcuboid@scalex*sin(\tikzcuboid@anglex)} + \pgfmathsetmacro{\vectoryx}{\tikzcuboid@scaley*cos(\tikzcuboid@angley)} + \pgfmathsetmacro{\vectoryy}{\tikzcuboid@scaley*sin(\tikzcuboid@angley)} + \pgfmathsetmacro{\vectorzx}{\tikzcuboid@scalez*cos(\tikzcuboid@anglez)} + \pgfmathsetmacro{\vectorzy}{\tikzcuboid@scalez*sin(\tikzcuboid@anglez)} + \begin{scope}[xshift=\tikzcuboid@shiftx, yshift=\tikzcuboid@shifty, scale=\tikzcuboid@scale, rotate=\tikzcuboid@rotation, x={(\vectorxx,\vectorxy)}, y={(\vectoryx,\vectoryy)}, z={(\vectorzx,\vectorzy)}] + \pgfmathsetmacro{\steppingx}{1/\tikzcuboid@densityx} + \pgfmathsetmacro{\steppingy}{1/\tikzcuboid@densityy} + \pgfmathsetmacro{\steppingz}{1/\tikzcuboid@densityz} + \newcommand{\dimx}{\tikzcuboid@dimx} + \newcommand{\dimy}{\tikzcuboid@dimy} + \newcommand{\dimz}{\tikzcuboid@dimz} + \pgfmathsetmacro{\secondx}{2*\steppingx} + \pgfmathsetmacro{\secondy}{2*\steppingy} + \pgfmathsetmacro{\secondz}{2*\steppingz} + \foreach \x in {\steppingx,\secondx,...,\dimx} + { \foreach \y in {\steppingy,\secondy,...,\dimy} + { \pgfmathsetmacro{\lowx}{(\x-\steppingx)} + \pgfmathsetmacro{\lowy}{(\y-\steppingy)} + \filldraw[fill=\tikzcuboid@fillfront,draw=\tikzcuboid@linefront] (\lowx,\lowy,\dimz) -- (\lowx,\y,\dimz) -- (\x,\y,\dimz) -- (\x,\lowy,\dimz) -- cycle; + + } + } + \foreach \x in {\steppingx,\secondx,...,\dimx} + { \foreach \z in {\steppingz,\secondz,...,\dimz} + { \pgfmathsetmacro{\lowx}{(\x-\steppingx)} + \pgfmathsetmacro{\lowz}{(\z-\steppingz)} + \filldraw[fill=\tikzcuboid@filltop,draw=\tikzcuboid@linetop] (\lowx,\dimy,\lowz) -- (\lowx,\dimy,\z) -- (\x,\dimy,\z) -- (\x,\dimy,\lowz) -- cycle; + } + } + \foreach \y in {\steppingy,\secondy,...,\dimy} + { \foreach \z in {\steppingz,\secondz,...,\dimz} + { \pgfmathsetmacro{\lowy}{(\y-\steppingy)} + \pgfmathsetmacro{\lowz}{(\z-\steppingz)} + \filldraw[fill=\tikzcuboid@fillright,draw=\tikzcuboid@lineright] (\dimx,\lowy,\lowz) -- (\dimx,\lowy,\z) -- (\dimx,\y,\z) -- (\dimx,\y,\lowz) -- cycle; + } + } + \ifthenelse{\equal{\tikzcuboid@emphedge}{Y}}% + {\draw[\tikzcuboid@emphstyle](0,\dimy,0) -- (\dimx,\dimy,0) -- (\dimx,\dimy,\dimz) -- (0,\dimy,\dimz) -- cycle;% + \draw[\tikzcuboid@emphstyle] (0,0,\dimz) -- (0,\dimy,\dimz) -- (\dimx,\dimy,\dimz) -- (\dimx,0,\dimz) -- cycle;% + \draw[\tikzcuboid@emphstyle](\dimx,0,0) -- (\dimx,\dimy,0) -- (\dimx,\dimy,\dimz) -- (\dimx,0,\dimz) -- cycle;% + }% + {} + \end{scope} +} + +\makeatother + +\begin{document} + +\begin{tikzpicture} + \tikzcuboid{% + shiftx=21cm,% + shifty=8cm,% + scale=1.00,% + rotation=0,% + densityx=2,% + densityy=2,% + densityz=2,% + dimx=4,% + dimy=3,% + dimz=3,% + linefront=purple!75!black,% + linetop=purple!50!black,% + lineright=purple!25!black,% + fillfront=purple!25!white,% + filltop=purple!50!white,% + fillright=purple!75!white,% + emphedge=Y,% + emphstyle=ultra thick, + } + \tikzcuboid{% + shiftx=21cm,% + shifty=11.6cm,% + scale=1.00,% + rotation=0,% + densityx=2,% + densityy=2,% + densityz=2,% + dimx=4,% + dimy=3,% + dimz=3,% + linefront=teal!75!black,% + linetop=teal!50!black,% + lineright=teal!25!black,% + fillfront=teal!25!white,% + filltop=teal!50!white,% + fillright=teal!75!white,% + emphedge=Y,% + emphstyle=ultra thick, + } + \tikzcuboid{% + shiftx=26.8cm,% + shifty=8cm,% + scale=1.00,% + rotation=0,% + densityx=10000,% + densityy=2,% + densityz=2,% + dimx=0,% + dimy=3,% + dimz=3,% + linefront=orange!75!black,% + linetop=orange!50!black,% + lineright=orange!25!black,% + fillfront=orange!25!white,% + filltop=orange!50!white,% + fillright=orange!100!white,% + emphedge=Y,% + emphstyle=ultra thick, + } + \tikzcuboid{% + shiftx=28.6cm,% + shifty=8cm,% + scale=1.00,% + rotation=0,% + densityx=10000,% + densityy=2,% + densityz=2,% + dimx=0,% + dimy=3,% + dimz=3,% + linefront=purple!75!black,% + linetop=purple!50!black,% + lineright=purple!25!black,% + fillfront=purple!25!white,% + filltop=purple!50!white,% + fillright=red!75!white,% + emphedge=Y,% + emphstyle=ultra thick, + } + % \tikzcuboid{% + % shiftx=27.1cm,% + % shifty=10.1cm,% + % scale=1.00,% + % rotation=0,% + % densityx=100,% + % densityy=2,% + % densityz=100,% + % dimx=0,% + % dimy=3,% + % dimz=0,% + % emphedge=Y,% + % emphstyle=ultra thick, + % } + % \tikzcuboid{% + % shiftx=27.1cm,% + % shifty=10.1cm,% + % scale=1.00,% + % rotation=180,% + % densityx=100,% + % densityy=100,% + % densityz=2,% + % dimx=0,% + % dimy=0,% + % dimz=3,% + % emphedge=Y,% + % emphstyle=ultra thick, + % } + \tikzcuboid{% + shiftx=26.8cm,% + shifty=11.4cm,% + scale=1.00,% + rotation=0,% + densityx=100,% + densityy=2,% + densityz=100,% + dimx=0,% + dimy=3,% + dimz=0,% + emphedge=Y,% + emphstyle=ultra thick, + } + \tikzcuboid{% + shiftx=25.3cm,% + shifty=12.9cm,% + scale=1.00,% + rotation=180,% + densityx=100,% + densityy=100,% + densityz=2,% + dimx=0,% + dimy=0,% + dimz=3,% + emphedge=Y,% + emphstyle=ultra thick, + } + % \fill (27.1,10.1) circle[radius=2pt]; + \node [font=\fontsize{130}{100}\fontfamily{phv}\selectfont, anchor=east, text width=2cm, align=right, color=white!50!black] at (19.8,4.4) {\textbf{\emph{x}}}; + \node [font=\fontsize{130}{100}\fontfamily{phv}\selectfont, anchor=west, text width=10cm, align=left] at (20.3,4) {{array}}; +\end{tikzpicture} + +\end{document} From afee350789cacecc689c63fe580c98025e29d3db Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 14 Jul 2018 11:19:52 -0500 Subject: [PATCH 163/282] fix dask get_scheduler warning (#2282) * fix dask get_scheduler warning * test + whatsnew * skip test when we cant write netcdfs/ --- doc/whats-new.rst | 4 ++++ xarray/backends/api.py | 16 ++++++------- xarray/backends/common.py | 42 ++++++++++++++++++++--------------- xarray/tests/test_backends.py | 9 ++++++++ 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index af90ea7f9d3..dc1349dcd81 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -66,6 +66,10 @@ Bug fixes weren't monotonic (:issue:`2250`). By `Fabien Maussion `_. +- Fixed warning raised in :py:meth:`~Dataset.to_netcdf` due to deprecation of + `effective_get` in dask (:issue:`2238`). + By `Joe Hamman `_. + Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/xarray/backends/api.py b/xarray/backends/api.py index d5e2e8bbc2c..b2c0df7b01b 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -13,7 +13,7 @@ from ..core.pycompat import basestring, path_type from ..core.utils import close_on_error, is_remote_uri from .common import ( - HDF5_LOCK, ArrayWriter, CombinedLock, get_scheduler, get_scheduler_lock) + HDF5_LOCK, ArrayWriter, CombinedLock, _get_scheduler, _get_scheduler_lock) DATAARRAY_NAME = '__xarray_dataarray_name__' DATAARRAY_VARIABLE = '__xarray_dataarray_variable__' @@ -136,7 +136,7 @@ def _get_lock(engine, scheduler, format, path_or_file): locks = [] if format in ['NETCDF4', None] and engine in ['h5netcdf', 'netcdf4']: locks.append(HDF5_LOCK) - locks.append(get_scheduler_lock(scheduler, path_or_file)) + locks.append(_get_scheduler_lock(scheduler, path_or_file)) # When we have more than one lock, use the CombinedLock wrapper class lock = CombinedLock(locks) if len(locks) > 1 else locks[0] @@ -179,7 +179,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, taken from variable attributes (if they exist). If the `_FillValue` or `missing_value` attribute contains multiple values a warning will be issued and all array values matching one of the multiple values will - be replaced by NA. mask_and_scale defaults to True except for the + be replaced by NA. mask_and_scale defaults to True except for the pseudonetcdf backend. decode_times : bool, optional If True, decode times encoded in the standard NetCDF datetime format @@ -223,7 +223,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, inconsistent values. backend_kwargs: dictionary, optional A dictionary of keyword arguments to pass on to the backend. This - may be useful when backend options would improve performance or + may be useful when backend options would improve performance or allow user control of dataset processing. Returns @@ -235,7 +235,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, -------- open_mfdataset """ - + if mask_and_scale is None: mask_and_scale = not engine == 'pseudonetcdf' @@ -385,7 +385,7 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, taken from variable attributes (if they exist). If the `_FillValue` or `missing_value` attribute contains multiple values a warning will be issued and all array values matching one of the multiple values will - be replaced by NA. mask_and_scale defaults to True except for the + be replaced by NA. mask_and_scale defaults to True except for the pseudonetcdf backend. decode_times : bool, optional If True, decode times encoded in the standard NetCDF datetime format @@ -428,7 +428,7 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, inconsistent values. backend_kwargs: dictionary, optional A dictionary of keyword arguments to pass on to the backend. This - may be useful when backend options would improve performance or + may be useful when backend options would improve performance or allow user control of dataset processing. Notes @@ -699,7 +699,7 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, sync = writer is None # handle scheduler specific logic - scheduler = get_scheduler() + scheduler = _get_scheduler() have_chunks = any(v.chunks for v in dataset.variables.values()) if (have_chunks and scheduler in ['distributed', 'multiprocessing'] and engine != 'netcdf4'): diff --git a/xarray/backends/common.py b/xarray/backends/common.py index d5eccd9be52..99f7698ee92 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -31,36 +31,42 @@ NONE_VAR_NAME = '__values__' -def get_scheduler(get=None, collection=None): +def _get_scheduler(get=None, collection=None): """ Determine the dask scheduler that is being used. None is returned if not dask scheduler is active. See also -------- - dask.utils.effective_get + dask.base.get_scheduler """ try: - from dask.utils import effective_get - actual_get = effective_get(get, collection) + # dask 0.18.1 and later + from dask.base import get_scheduler + actual_get = get_scheduler(get, collection) + except ImportError: try: - from dask.distributed import Client - if isinstance(actual_get.__self__, Client): - return 'distributed' - except (ImportError, AttributeError): - try: - import dask.multiprocessing - if actual_get == dask.multiprocessing.get: - return 'multiprocessing' - else: - return 'threaded' - except ImportError: + from dask.utils import effective_get + actual_get = effective_get(get, collection) + except ImportError: + return None + + try: + from dask.distributed import Client + if isinstance(actual_get.__self__, Client): + return 'distributed' + except (ImportError, AttributeError): + try: + import dask.multiprocessing + if actual_get == dask.multiprocessing.get: + return 'multiprocessing' + else: return 'threaded' - except ImportError: - return None + except ImportError: + return 'threaded' -def get_scheduler_lock(scheduler, path_or_file=None): +def _get_scheduler_lock(scheduler, path_or_file=None): """ Get the appropriate lock for a certain situation based onthe dask scheduler used. diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 9ec68bb0846..b0928809cff 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -3323,3 +3323,12 @@ def test_pickle_reconstructor(): obj3 = pickle.loads(p_obj2) assert obj3.value.readlines() == lines + + +@requires_scipy_or_netCDF4 +def test_no_warning_from_dask_effective_get(): + with create_tmp_file() as tmpfile: + with pytest.warns(None) as record: + ds = Dataset() + ds.to_netcdf(tmpfile) + assert len(record) == 0 From df63fce69b9cde68689362895ec394df75335313 Mon Sep 17 00:00:00 2001 From: Yohai Bar Sinai <6164157+yohai@users.noreply.github.com> Date: Tue, 17 Jul 2018 16:01:27 -0400 Subject: [PATCH 164/282] DOC: replace broken link by a link to @shoyer's personal blog (#2296) --- doc/dask.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/dask.rst b/doc/dask.rst index 2d4beea4f70..672450065cb 100644 --- a/doc/dask.rst +++ b/doc/dask.rst @@ -13,7 +13,7 @@ dependency in a future version of xarray. For a full example of how to use xarray's dask integration, read the `blog post introducing xarray and dask`_. -.. _blog post introducing xarray and dask: https://www.anaconda.com/blog/developer-blog/xray-dask-out-core-labeled-arrays-python/ +.. _blog post introducing xarray and dask: http://stephanhoyer.com/2015/06/11/xray-dask-out-of-core-labeled-arrays/ What is a dask array? --------------------- From 74625d463ec71ac6f4a3642d3385bd9cbf8381ce Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 18 Jul 2018 08:43:25 -0700 Subject: [PATCH 165/282] Update minimum dependencies in setup.py --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e35611e01b1..88c27c95118 100644 --- a/setup.py +++ b/setup.py @@ -20,13 +20,12 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Topic :: Scientific/Engineering', ] -INSTALL_REQUIRES = ['numpy >= 1.11', 'pandas >= 0.18.0'] +INSTALL_REQUIRES = ['numpy >= 1.12', 'pandas >= 0.19.2'] TESTS_REQUIRE = ['pytest >= 2.7.1'] if sys.version_info[0] < 3: TESTS_REQUIRE.append('mock') From 024aa098b1d385d806ad7e3b97bcd3a5033f2ae0 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 18 Jul 2018 08:45:44 -0700 Subject: [PATCH 166/282] Release v0.10.8 --- doc/installing.rst | 1 + doc/whats-new.rst | 31 ++++++++++++++----------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/doc/installing.rst b/doc/installing.rst index b3154c3d8bb..cdb1da44ef6 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -101,6 +101,7 @@ A fixed-point performance monitoring of (a part of) our codes can be seen on `this page `__. To run these benchmark tests in a local machine, first install + - `airspeed-velocity `__: a tool for benchmarking Python packages over their lifetime. and run diff --git a/doc/whats-new.rst b/doc/whats-new.rst index dc1349dcd81..9d2a0a17789 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,11 +27,21 @@ What's New .. _whats-new.0.10.8: -v0.10.8 (unreleased) --------------------- +v0.10.8 (18 July 2018) +---------------------- -Documentation -~~~~~~~~~~~~~ +Breaking changes +~~~~~~~~~~~~~~~~ + +- Xarray no longer supports python 3.4. Additionally, the minimum supported + versions of the following dependencies has been updated and/or clarified: + + - Pandas: 0.18 -> 0.19 + - NumPy: 1.11 -> 1.12 + - Dask: 0.9 -> 0.16 + - Matplotlib: unspecified -> 1.5 + + (:issue:`2204`). By `Joe Hamman `_. Enhancements ~~~~~~~~~~~~ @@ -70,19 +80,6 @@ Bug fixes `effective_get` in dask (:issue:`2238`). By `Joe Hamman `_. -Breaking changes -~~~~~~~~~~~~~~~~ - -- Xarray no longer supports python 3.4. Additionally, the minimum supported - versions of the following dependencies has been updated and/or clarified: - - - Pandas: 0.18 -> 0.19 - - NumPy: 1.11 -> 1.12 - - Dask: 0.9 -> 0.16 - - Matplotlib: unspecified -> 1.5 - - (:issue:`2204`). By `Joe Hamman `_. - .. _whats-new.0.10.7: v0.10.7 (7 June 2018) From c77e8b12c522853ba1df43057a66422a22a9b688 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 18 Jul 2018 08:52:59 -0700 Subject: [PATCH 167/282] revert to dev ersion --- doc/whats-new.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 9d2a0a17789..48be14f6d50 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,21 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ +.. _whats-new.0.10.9: + +v0.10.9 (unreleased) +-------------------- + +Documentation +~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + + .. _whats-new.0.10.8: v0.10.8 (18 July 2018) From b5a8d86bc174694bc48cc3ddd9966f131d26f7bf Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 18 Jul 2018 08:58:23 -0700 Subject: [PATCH 168/282] Simplify release checklist --- HOW_TO_RELEASE | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/HOW_TO_RELEASE b/HOW_TO_RELEASE index cdfcace809a..80f37e672a5 100644 --- a/HOW_TO_RELEASE +++ b/HOW_TO_RELEASE @@ -14,6 +14,7 @@ Time required: about an hour. 5. Tag the release: git tag -a v0.X.Y -m 'v0.X.Y' 6. Build source and binary wheels for pypi: + git clean -xdf # this deletes all uncommited changes! python setup.py bdist_wheel sdist 7. Use twine to register and upload the release on pypi. Be careful, you can't take this back! @@ -37,16 +38,12 @@ Time required: about an hour. git push upstream master You're done pushing to master! 12. Issue the release on GitHub. Click on "Draft a new release" at - https://github.com/pydata/xarray/releases and paste in the latest from - whats-new.rst. + https://github.com/pydata/xarray/releases. Type in the version number, but + don't bother to describe it -- we maintain that on the docs instead. 13. Update the docs. Login to https://readthedocs.org/projects/xray/versions/ and switch your new release tag (at the bottom) from "Inactive" to "Active". It should now build automatically. -14. Update conda-forge. Clone https://github.com/conda-forge/xarray-feedstock - and update the version number and sha256 in meta.yaml. (On OS X, you can - calculate sha256 with `shasum -a 256 xarray-0.X.Y.tar.gz`). Submit a pull - request (and merge it, once CI passes). -15. Issue the release announcement! For bug fix releases, I usually only email +14. Issue the release announcement! For bug fix releases, I usually only email xarray@googlegroups.com. For major/feature releases, I will email a broader list (no more than once every 3-6 months): pydata@googlegroups.com, xarray@googlegroups.com, From 7cd3442fc61e94601c3bfb20377f4f795cde584d Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 18 Jul 2018 20:50:29 -0700 Subject: [PATCH 169/282] Add indexing.explicit_indexing_adapter and refactor backends to use it. (#2297) * Add indexing.explicit_indexing_adapter and refactor backends to use it. No external API changes. This simplifies the boilerplate for supporting explicit indexing in a backend array class. CC fujiisoup * lint --- xarray/backends/h5netcdf_.py | 16 +++++------ xarray/backends/netCDF4_.py | 13 +++++---- xarray/backends/pseudonetcdf_.py | 13 ++++----- xarray/backends/pydap_.py | 12 ++++----- xarray/backends/pynio_.py | 13 ++++----- xarray/backends/rasterio_.py | 46 +++++++++++++++++++------------- xarray/coding/variables.py | 5 +++- xarray/core/computation.py | 3 ++- xarray/core/indexing.py | 31 +++++++++++++++++++++ xarray/tests/test_backends.py | 6 ++++- 10 files changed, 96 insertions(+), 62 deletions(-) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index ecc83e98691..959cd221734 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -17,20 +17,16 @@ class H5NetCDFArrayWrapper(BaseNetCDF4Array): def __getitem__(self, key): - key, np_inds = indexing.decompose_indexer( - key, self.shape, indexing.IndexingSupport.OUTER_1VECTOR) + return indexing.explicit_indexing_adapter( + key, self.shape, indexing.IndexingSupport.OUTER_1VECTOR, + self._getitem) + def _getitem(self, key): # h5py requires using lists for fancy indexing: # https://github.com/h5py/h5py/issues/992 - key = tuple(list(k) if isinstance(k, np.ndarray) else k for k in - key.tuple) + key = tuple(list(k) if isinstance(k, np.ndarray) else k for k in key) with self.datastore.ensure_open(autoclose=True): - array = self.get_array()[key] - - if len(np_inds.tuple) > 0: - array = indexing.NumpyIndexingAdapter(array)[np_inds] - - return array + return self.get_array()[key] def maybe_decode_bytes(txt): diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index d26b2b5321e..5c6d82fd126 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -54,8 +54,11 @@ def get_array(self): class NetCDF4ArrayWrapper(BaseNetCDF4Array): def __getitem__(self, key): - key, np_inds = indexing.decompose_indexer( - key, self.shape, indexing.IndexingSupport.OUTER) + return indexing.explicit_indexing_adapter( + key, self.shape, indexing.IndexingSupport.OUTER, + self._getitem) + + def _getitem(self, key): if self.datastore.is_remote: # pragma: no cover getitem = functools.partial(robust_getitem, catch=RuntimeError) else: @@ -63,7 +66,7 @@ def __getitem__(self, key): with self.datastore.ensure_open(autoclose=True): try: - array = getitem(self.get_array(), key.tuple) + array = getitem(self.get_array(), key) except IndexError: # Catch IndexError in netCDF4 and return a more informative # error message. This is most often called when an unsorted @@ -75,10 +78,6 @@ def __getitem__(self, key): import traceback msg += '\n\nOriginal traceback:\n' + traceback.format_exc() raise IndexError(msg) - - if len(np_inds.tuple) > 0: - array = indexing.NumpyIndexingAdapter(array)[np_inds] - return array diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index c481bf848b9..d946c6fa927 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -28,16 +28,13 @@ def get_array(self): return self.datastore.ds.variables[self.variable_name] def __getitem__(self, key): - key, np_inds = indexing.decompose_indexer( - key, self.shape, indexing.IndexingSupport.OUTER_1VECTOR) + return indexing.explicit_indexing_adapter( + key, self.shape, indexing.IndexingSupport.OUTER_1VECTOR, + self._getitem) + def _getitem(self, key): with self.datastore.ensure_open(autoclose=True): - array = self.get_array()[key.tuple] # index backend array - - if len(np_inds.tuple) > 0: - # index the loaded np.ndarray - array = indexing.NumpyIndexingAdapter(array)[np_inds] - return array + return self.get_array()[key] class PseudoNetCDFDataStore(AbstractDataStore, DataStorePickleMixin): diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index 4a932e3dad2..71ea4841b71 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -22,22 +22,20 @@ def dtype(self): return self.array.dtype def __getitem__(self, key): - key, np_inds = indexing.decompose_indexer( - key, self.shape, indexing.IndexingSupport.BASIC) + return indexing.explicit_indexing_adapter( + key, self.shape, indexing.IndexingSupport.BASIC, self._getitem) + def _getitem(self, key): # pull the data from the array attribute if possible, to avoid # downloading coordinate data twice array = getattr(self.array, 'array', self.array) - result = robust_getitem(array, key.tuple, catch=ValueError) + result = robust_getitem(array, key, catch=ValueError) # pydap doesn't squeeze axes automatically like numpy - axis = tuple(n for n, k in enumerate(key.tuple) + axis = tuple(n for n, k in enumerate(key) if isinstance(k, integer_types)) if len(axis) > 0: result = np.squeeze(result, axis) - if len(np_inds.tuple) > 0: - result = indexing.NumpyIndexingAdapter(np.asarray(result))[np_inds] - return result diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 3c638b6b057..98b76928597 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -24,19 +24,16 @@ def get_array(self): return self.datastore.ds.variables[self.variable_name] def __getitem__(self, key): - key, np_inds = indexing.decompose_indexer( - key, self.shape, indexing.IndexingSupport.BASIC) + return indexing.explicit_indexing_adapter( + key, self.shape, indexing.IndexingSupport.BASIC, self._getitem) + def _getitem(self, key): with self.datastore.ensure_open(autoclose=True): array = self.get_array() - if key.tuple == () and self.ndim == 0: + if key == () and self.ndim == 0: return array.get_value() - array = array[key.tuple] - if len(np_inds.tuple) > 0: - array = indexing.NumpyIndexingAdapter(array)[np_inds] - - return array + return array[key] class NioDataStore(AbstractDataStore, DataStorePickleMixin): diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 0f19a1b51be..e576435fcd2 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -47,7 +47,7 @@ def _get_indexer(self, key): Parameter --------- - key: ExplicitIndexer + key: tuple of int Returns ------- @@ -60,13 +60,11 @@ def _get_indexer(self, key): -------- indexing.decompose_indexer """ - key, np_inds = indexing.decompose_indexer( - key, self.shape, indexing.IndexingSupport.OUTER) + assert len(key) == 3, 'rasterio datasets should always be 3D' # bands cannot be windowed but they can be listed - band_key = key.tuple[0] - new_shape = [] - np_inds2 = [] + band_key = key[0] + np_inds = [] # bands (axis=0) cannot be windowed but they can be listed if isinstance(band_key, slice): start, stop, step = band_key.indices(self.shape[0]) @@ -74,18 +72,16 @@ def _get_indexer(self, key): # be sure we give out a list band_key = (np.asarray(band_key) + 1).tolist() if isinstance(band_key, list): # if band_key is not a scalar - new_shape.append(len(band_key)) - np_inds2.append(slice(None)) + np_inds.append(slice(None)) # but other dims can only be windowed window = [] squeeze_axis = [] - for i, (k, n) in enumerate(zip(key.tuple[1:], self.shape[1:])): + for i, (k, n) in enumerate(zip(key[1:], self.shape[1:])): if isinstance(k, slice): # step is always positive. see indexing.decompose_indexer start, stop, step = k.indices(n) - np_inds2.append(slice(None, None, step)) - new_shape.append(stop - start) + np_inds.append(slice(None, None, step)) elif is_scalar(k): # windowed operations will always return an array # we will have to squeeze it later @@ -94,21 +90,33 @@ def _get_indexer(self, key): stop = k + 1 else: start, stop = np.min(k), np.max(k) + 1 - np_inds2.append(k - start) - new_shape.append(stop - start) + np_inds.append(k - start) window.append((start, stop)) - np_inds = indexing._combine_indexers( - indexing.OuterIndexer(tuple(np_inds2)), new_shape, np_inds) - return band_key, window, tuple(squeeze_axis), np_inds + if isinstance(key[1], np.ndarray) and isinstance(key[2], np.ndarray): + # do outer-style indexing + np_inds[1:] = np.ix_(*np_inds[1:]) - def __getitem__(self, key): + return band_key, tuple(window), tuple(squeeze_axis), tuple(np_inds) + + def _getitem(self, key): band_key, window, squeeze_axis, np_inds = self._get_indexer(key) - out = self.riods.value.read(band_key, window=tuple(window)) + if not band_key or any(start == stop for (start, stop) in window): + # no need to do IO + shape = (len(band_key),) + tuple( + stop - start for (start, stop) in window) + out = np.zeros(shape, dtype=self.dtype) + else: + out = self.riods.value.read(band_key, window=window) + if squeeze_axis: out = np.squeeze(out, axis=squeeze_axis) - return indexing.NumpyIndexingAdapter(out)[np_inds] + return out[np_inds] + + def __getitem__(self, key): + return indexing.explicit_indexing_adapter( + key, self.shape, indexing.IndexingSupport.OUTER, self._getitem) def _parse_envi(meta): diff --git a/xarray/coding/variables.py b/xarray/coding/variables.py index 1207f5743cb..b86b77a3707 100644 --- a/xarray/coding/variables.py +++ b/xarray/coding/variables.py @@ -63,7 +63,10 @@ def dtype(self): return np.dtype(self._dtype) def __getitem__(self, key): - return self.func(self.array[key]) + return type(self)(self.array[key], self.func, self.dtype) + + def __array__(self, dtype=None): + return self.func(self.array) def __repr__(self): return ("%s(%r, func=%r, dtype=%r)" % diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 9b251bb2c4b..1f771a4103c 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -10,7 +10,8 @@ import numpy as np -from . import duck_array_ops, utils +from . import duck_array_ops +from . import utils from .alignment import deep_align from .merge import expand_and_merge_variables from .pycompat import OrderedDict, dask_array_type, basestring diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 2c1f08379ab..d51da471c8d 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -749,6 +749,37 @@ class IndexingSupport(object): # could inherit from enum.Enum on Python 3 VECTORIZED = 'VECTORIZED' +def explicit_indexing_adapter( + key, shape, indexing_support, raw_indexing_method): + """Support explicit indexing by delegating to a raw indexing method. + + Outer and/or vectorized indexers are supported by indexing a second time + with a NumPy array. + + Parameters + ---------- + key : ExplicitIndexer + Explicit indexing object. + shape : Tuple[int, ...] + Shape of the indexed array. + indexing_support : IndexingSupport enum + Form of indexing supported by raw_indexing_method. + raw_indexing_method: callable + Function (like ndarray.__getitem__) that when called with indexing key + in the form of a tuple returns an indexed array. + + Returns + ------- + Indexing result, in the form of a duck numpy-array. + """ + raw_key, numpy_indices = decompose_indexer(key, shape, indexing_support) + result = raw_indexing_method(raw_key.tuple) + if numpy_indices.tuple: + # index the loaded np.ndarray + result = NumpyIndexingAdapter(np.asarray(result))[numpy_indices] + return result + + def decompose_indexer(indexer, shape, indexing_support): if isinstance(indexer, VectorizedIndexer): return _decompose_vectorized_indexer(indexer, shape, indexing_support) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index b0928809cff..c90ca01cc11 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2932,12 +2932,16 @@ def test_indexing(self): assert_allclose(expected.isel(**ind), actual.isel(**ind)) assert not actual.variable._in_memory - # None is selected + # empty selection ind = {'band': np.array([2, 1, 0]), 'x': 1, 'y': slice(2, 2, 1)} assert_allclose(expected.isel(**ind), actual.isel(**ind)) assert not actual.variable._in_memory + ind = {'band': slice(0, 0), 'x': 1, 'y': 2} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + # vectorized indexer ind = {'band': DataArray([2, 1, 0], dims='a'), 'x': DataArray([1, 0, 0], dims='a'), From 9ecd406347738f222e835ca719d5f4d46e6ebd1a Mon Sep 17 00:00:00 2001 From: tv3141 Date: Thu, 19 Jul 2018 06:56:23 +0100 Subject: [PATCH 170/282] iris conversion (#2202) * Add tests. * Add var_name fallback. * Add fall back to AuxCoord. * Remove unnecesary import. * Add var_name fallback tests to existing tests * Add tests for fallback to AuxCoord to existing tests. * var_name before standard_name to pass tests * make tests more explicit * delete new test file * cleanup code * No name for DataArray if cube is nameless. * No unknown unit entry for DataArray coords. * Use name resolution similar to iris.obj.name(). * Add check for duplicate dimension names. * Cleanup: remove duplicate tests. * Add test for default name. * Add edge case test. * Test for cube name usage. * Describe changes in whats-new.rst * Create Iris conversion test class. * Replace TestCase methods. * parametrized test for cube name * parametrized test for coord name * separate test for duplicate coord names * separate test for AuxCoord fallback * Refactor iris availability check. * cleanup * Make type comparison PEP8 compliant. * Keep similar tests close together. * Improve edge case comment * Remove trailing whitespace. * Move import to top. * Remove STASH lookup. * Changelog into next release. * Fix long line after rebase. * Remove duplicate test after rebase. --- doc/whats-new.rst | 5 + xarray/convert.py | 41 ++-- xarray/tests/__init__.py | 1 + xarray/tests/test_dataarray.py | 356 ++++++++++++++++++++------------- 4 files changed, 248 insertions(+), 155 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 48be14f6d50..e392e3ce869 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -39,6 +39,11 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed ``DataArray.to_iris()`` failure while creating ``DimCoord`` by + falling back to creating ``AuxCoord``. Fixed dependency on ``var_name`` + attribute being set. + (:issue:`2201`) + By `Thomas Voigt `_. .. _whats-new.0.10.8: diff --git a/xarray/convert.py b/xarray/convert.py index f3a5ccb2ce5..bc639c68455 100644 --- a/xarray/convert.py +++ b/xarray/convert.py @@ -2,7 +2,8 @@ """ from __future__ import absolute_import, division, print_function -from collections import OrderedDict +from collections import Counter, OrderedDict + import numpy as np import pandas as pd @@ -156,8 +157,12 @@ def to_iris(dataarray): if coord.dims: axis = dataarray.get_axis_num(coord.dims) if coord_name in dataarray.dims: - iris_coord = iris.coords.DimCoord(coord.values, **coord_args) - dim_coords.append((iris_coord, axis)) + try: + iris_coord = iris.coords.DimCoord(coord.values, **coord_args) + dim_coords.append((iris_coord, axis)) + except ValueError: + iris_coord = iris.coords.AuxCoord(coord.values, **coord_args) + aux_coords.append((iris_coord, axis)) else: iris_coord = iris.coords.AuxCoord(coord.values, **coord_args) aux_coords.append((iris_coord, axis)) @@ -183,7 +188,7 @@ def _iris_obj_to_attrs(obj): 'long_name': obj.long_name} if obj.units.calendar: attrs['calendar'] = obj.units.calendar - if obj.units.origin != '1': + if obj.units.origin != '1' and not obj.units.is_unknown(): attrs['units'] = obj.units.origin attrs.update(obj.attributes) return dict((k, v) for k, v in attrs.items() if v is not None) @@ -206,34 +211,46 @@ def _iris_cell_methods_to_str(cell_methods_obj): return ' '.join(cell_methods) +def _name(iris_obj, default='unknown'): + """ Mimicks `iris_obj.name()` but with different name resolution order. + + Similar to iris_obj.name() method, but using iris_obj.var_name first to + enable roundtripping. + """ + return (iris_obj.var_name or iris_obj.standard_name or + iris_obj.long_name or default) + + def from_iris(cube): """ Convert a Iris cube into an DataArray """ import iris.exceptions from xarray.core.pycompat import dask_array_type - name = cube.var_name + name = _name(cube) + if name == 'unknown': + name = None dims = [] for i in range(cube.ndim): try: dim_coord = cube.coord(dim_coords=True, dimensions=(i,)) - dims.append(dim_coord.var_name) + dims.append(_name(dim_coord)) except iris.exceptions.CoordinateNotFoundError: dims.append("dim_{}".format(i)) + if len(set(dims)) != len(dims): + duplicates = [k for k, v in Counter(dims).items() if v > 1] + raise ValueError('Duplicate coordinate name {}.'.format(duplicates)) + coords = OrderedDict() for coord in cube.coords(): coord_attrs = _iris_obj_to_attrs(coord) coord_dims = [dims[i] for i in cube.coord_dims(coord)] - if not coord.var_name: - raise ValueError("Coordinate '{}' has no " - "var_name attribute".format(coord.name())) if coord_dims: - coords[coord.var_name] = (coord_dims, coord.points, coord_attrs) + coords[_name(coord)] = (coord_dims, coord.points, coord_attrs) else: - coords[coord.var_name] = ((), - np.asscalar(coord.points), coord_attrs) + coords[_name(coord)] = ((), np.asscalar(coord.points), coord_attrs) array_attrs = _iris_obj_to_attrs(cube) cell_methods = _iris_cell_methods_to_str(cube.cell_methods) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 3b4d69a35f7..33a8da6bbfb 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -74,6 +74,7 @@ def _importorskip(modname, minversion=None): has_pathlib, requires_pathlib = _importorskip('pathlib') has_zarr, requires_zarr = _importorskip('zarr', minversion='2.2') has_np113, requires_np113 = _importorskip('numpy', minversion='1.13.0') +has_iris, requires_iris = _importorskip('iris') # some special cases has_scipy_or_netCDF4 = has_scipy or has_netCDF4 diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index e0b1496c7bf..fdca3fd8d13 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -19,7 +19,8 @@ from xarray.tests import ( ReturnItem, TestCase, assert_allclose, assert_array_equal, assert_equal, assert_identical, raises_regex, requires_bottleneck, requires_cftime, - requires_dask, requires_np113, requires_scipy, source_ndarray, unittest) + requires_dask, requires_iris, requires_np113, requires_scipy, + source_ndarray, unittest) class TestDataArray(TestCase): @@ -2994,148 +2995,6 @@ def test_to_and_from_cdms2_ugrid(self): assert_array_equal(original.coords['lat'], back.coords['lat']) assert_array_equal(original.coords['lon'], back.coords['lon']) - def test_to_and_from_iris(self): - try: - import iris - import cf_units - except ImportError: - raise unittest.SkipTest('iris not installed') - - coord_dict = OrderedDict() - coord_dict['distance'] = ('distance', [-2, 2], {'units': 'meters'}) - coord_dict['time'] = ('time', pd.date_range('2000-01-01', periods=3)) - coord_dict['height'] = 10 - coord_dict['distance2'] = ('distance', [0, 1], {'foo': 'bar'}) - coord_dict['time2'] = (('distance', 'time'), [[0, 1, 2], [2, 3, 4]]) - - original = DataArray(np.arange(6, dtype='float').reshape(2, 3), - coord_dict, name='Temperature', - attrs={'baz': 123, 'units': 'Kelvin', - 'standard_name': 'fire_temperature', - 'long_name': 'Fire Temperature'}, - dims=('distance', 'time')) - - # Set a bad value to test the masking logic - original.data[0, 2] = np.NaN - - original.attrs['cell_methods'] = \ - 'height: mean (comment: A cell method)' - actual = original.to_iris() - assert_array_equal(actual.data, original.data) - assert actual.var_name == original.name - self.assertItemsEqual([d.var_name for d in actual.dim_coords], - original.dims) - assert (actual.cell_methods == (iris.coords.CellMethod( - method='mean', - coords=('height', ), - intervals=(), - comments=('A cell method', )), )) - - for coord, orginal_key in zip((actual.coords()), original.coords): - original_coord = original.coords[orginal_key] - assert coord.var_name == original_coord.name - assert_array_equal( - coord.points, CFDatetimeCoder().encode(original_coord).values) - assert (actual.coord_dims(coord) == - original.get_axis_num( - original.coords[coord.var_name].dims)) - - assert (actual.coord('distance2').attributes['foo'] == - original.coords['distance2'].attrs['foo']) - assert (actual.coord('distance').units == - cf_units.Unit(original.coords['distance'].units)) - assert actual.attributes['baz'] == original.attrs['baz'] - assert actual.standard_name == original.attrs['standard_name'] - - roundtripped = DataArray.from_iris(actual) - assert_identical(original, roundtripped) - - actual.remove_coord('time') - auto_time_dimension = DataArray.from_iris(actual) - assert auto_time_dimension.dims == ('distance', 'dim_1') - - actual.coord('distance').var_name = None - with raises_regex(ValueError, 'no var_name attribute'): - DataArray.from_iris(actual) - - @requires_dask - def test_to_and_from_iris_dask(self): - import dask.array as da - try: - import iris - import cf_units - except ImportError: - raise unittest.SkipTest('iris not installed') - - coord_dict = OrderedDict() - coord_dict['distance'] = ('distance', [-2, 2], {'units': 'meters'}) - coord_dict['time'] = ('time', pd.date_range('2000-01-01', periods=3)) - coord_dict['height'] = 10 - coord_dict['distance2'] = ('distance', [0, 1], {'foo': 'bar'}) - coord_dict['time2'] = (('distance', 'time'), [[0, 1, 2], [2, 3, 4]]) - - original = DataArray( - da.from_array(np.arange(-1, 5, dtype='float').reshape(2, 3), 3), - coord_dict, - name='Temperature', - attrs=dict(baz=123, units='Kelvin', - standard_name='fire_temperature', - long_name='Fire Temperature'), - dims=('distance', 'time')) - - # Set a bad value to test the masking logic - original.data = da.ma.masked_less(original.data, 0) - - original.attrs['cell_methods'] = \ - 'height: mean (comment: A cell method)' - actual = original.to_iris() - - # Be careful not to trigger the loading of the iris data - actual_data = actual.core_data() if \ - hasattr(actual, 'core_data') else actual.data - assert_array_equal(actual_data, original.data) - assert actual.var_name == original.name - self.assertItemsEqual([d.var_name for d in actual.dim_coords], - original.dims) - assert (actual.cell_methods == (iris.coords.CellMethod( - method='mean', - coords=('height', ), - intervals=(), - comments=('A cell method', )), )) - - for coord, orginal_key in zip((actual.coords()), original.coords): - original_coord = original.coords[orginal_key] - assert coord.var_name == original_coord.name - assert_array_equal( - coord.points, CFDatetimeCoder().encode(original_coord).values) - assert (actual.coord_dims(coord) == - original.get_axis_num( - original.coords[coord.var_name].dims)) - - assert (actual.coord('distance2').attributes['foo'] == original.coords[ - 'distance2'].attrs['foo']) - assert (actual.coord('distance').units == - cf_units.Unit(original.coords['distance'].units)) - assert actual.attributes['baz'] == original.attrs['baz'] - assert actual.standard_name == original.attrs['standard_name'] - - roundtripped = DataArray.from_iris(actual) - assert_identical(original, roundtripped) - - # If the Iris version supports it then we should have a dask array - # at each stage of the conversion - if hasattr(actual, 'core_data'): - self.assertEqual(type(original.data), type(actual.core_data())) - self.assertEqual(type(original.data), type(roundtripped.data)) - - actual.remove_coord('time') - auto_time_dimension = DataArray.from_iris(actual) - assert auto_time_dimension.dims == ('distance', 'dim_1') - - actual.coord('distance').var_name = None - with raises_regex(ValueError, 'no var_name attribute'): - DataArray.from_iris(actual) - def test_to_dataset_whole(self): unnamed = DataArray([1, 2], dims='x') with raises_regex(ValueError, 'unable to convert unnamed'): @@ -3693,3 +3552,214 @@ def test_raise_no_warning_for_nan_in_binary_ops(): with pytest.warns(None) as record: xr.DataArray([1, 2, np.NaN]) > 0 assert len(record) == 0 + + +class TestIrisConversion(object): + @requires_iris + def test_to_and_from_iris(self): + import iris + import cf_units # iris requirement + + # to iris + coord_dict = OrderedDict() + coord_dict['distance'] = ('distance', [-2, 2], {'units': 'meters'}) + coord_dict['time'] = ('time', pd.date_range('2000-01-01', periods=3)) + coord_dict['height'] = 10 + coord_dict['distance2'] = ('distance', [0, 1], {'foo': 'bar'}) + coord_dict['time2'] = (('distance', 'time'), [[0, 1, 2], [2, 3, 4]]) + + original = DataArray(np.arange(6, dtype='float').reshape(2, 3), + coord_dict, name='Temperature', + attrs={'baz': 123, 'units': 'Kelvin', + 'standard_name': 'fire_temperature', + 'long_name': 'Fire Temperature'}, + dims=('distance', 'time')) + + # Set a bad value to test the masking logic + original.data[0, 2] = np.NaN + + original.attrs['cell_methods'] = \ + 'height: mean (comment: A cell method)' + actual = original.to_iris() + assert_array_equal(actual.data, original.data) + assert actual.var_name == original.name + assert tuple(d.var_name for d in actual.dim_coords) == original.dims + assert (actual.cell_methods == (iris.coords.CellMethod( + method='mean', + coords=('height', ), + intervals=(), + comments=('A cell method', )), )) + + for coord, orginal_key in zip((actual.coords()), original.coords): + original_coord = original.coords[orginal_key] + assert coord.var_name == original_coord.name + assert_array_equal( + coord.points, CFDatetimeCoder().encode(original_coord).values) + assert (actual.coord_dims(coord) == + original.get_axis_num( + original.coords[coord.var_name].dims)) + + assert (actual.coord('distance2').attributes['foo'] == + original.coords['distance2'].attrs['foo']) + assert (actual.coord('distance').units == + cf_units.Unit(original.coords['distance'].units)) + assert actual.attributes['baz'] == original.attrs['baz'] + assert actual.standard_name == original.attrs['standard_name'] + + roundtripped = DataArray.from_iris(actual) + assert_identical(original, roundtripped) + + actual.remove_coord('time') + auto_time_dimension = DataArray.from_iris(actual) + assert auto_time_dimension.dims == ('distance', 'dim_1') + + @requires_iris + @requires_dask + def test_to_and_from_iris_dask(self): + import dask.array as da + import iris + import cf_units # iris requirement + + coord_dict = OrderedDict() + coord_dict['distance'] = ('distance', [-2, 2], {'units': 'meters'}) + coord_dict['time'] = ('time', pd.date_range('2000-01-01', periods=3)) + coord_dict['height'] = 10 + coord_dict['distance2'] = ('distance', [0, 1], {'foo': 'bar'}) + coord_dict['time2'] = (('distance', 'time'), [[0, 1, 2], [2, 3, 4]]) + + original = DataArray( + da.from_array(np.arange(-1, 5, dtype='float').reshape(2, 3), 3), + coord_dict, + name='Temperature', + attrs=dict(baz=123, units='Kelvin', + standard_name='fire_temperature', + long_name='Fire Temperature'), + dims=('distance', 'time')) + + # Set a bad value to test the masking logic + original.data = da.ma.masked_less(original.data, 0) + + original.attrs['cell_methods'] = \ + 'height: mean (comment: A cell method)' + actual = original.to_iris() + + # Be careful not to trigger the loading of the iris data + actual_data = actual.core_data() if \ + hasattr(actual, 'core_data') else actual.data + assert_array_equal(actual_data, original.data) + assert actual.var_name == original.name + assert tuple(d.var_name for d in actual.dim_coords) == original.dims + assert (actual.cell_methods == (iris.coords.CellMethod( + method='mean', + coords=('height', ), + intervals=(), + comments=('A cell method', )), )) + + for coord, orginal_key in zip((actual.coords()), original.coords): + original_coord = original.coords[orginal_key] + assert coord.var_name == original_coord.name + assert_array_equal( + coord.points, CFDatetimeCoder().encode(original_coord).values) + assert (actual.coord_dims(coord) == + original.get_axis_num( + original.coords[coord.var_name].dims)) + + assert (actual.coord('distance2').attributes['foo'] == original.coords[ + 'distance2'].attrs['foo']) + assert (actual.coord('distance').units == + cf_units.Unit(original.coords['distance'].units)) + assert actual.attributes['baz'] == original.attrs['baz'] + assert actual.standard_name == original.attrs['standard_name'] + + roundtripped = DataArray.from_iris(actual) + assert_identical(original, roundtripped) + + # If the Iris version supports it then we should have a dask array + # at each stage of the conversion + if hasattr(actual, 'core_data'): + assert isinstance(original.data, type(actual.core_data())) + assert isinstance(original.data, type(roundtripped.data)) + + actual.remove_coord('time') + auto_time_dimension = DataArray.from_iris(actual) + assert auto_time_dimension.dims == ('distance', 'dim_1') + + @requires_iris + @pytest.mark.parametrize('var_name, std_name, long_name, name, attrs', [ + ('var_name', 'height', 'Height', + 'var_name', {'standard_name': 'height', 'long_name': 'Height'}), + (None, 'height', 'Height', + 'height', {'standard_name': 'height', 'long_name': 'Height'}), + (None, None, 'Height', + 'Height', {'long_name': 'Height'}), + (None, None, None, + None, {}), + ]) + def test_da_name_from_cube(self, std_name, long_name, var_name, name, + attrs): + from iris.cube import Cube + + data = [] + cube = Cube(data, var_name=var_name, standard_name=std_name, + long_name=long_name) + result = xr.DataArray.from_iris(cube) + expected = xr.DataArray(data, name=name, attrs=attrs) + xr.testing.assert_identical(result, expected) + + @requires_iris + @pytest.mark.parametrize('var_name, std_name, long_name, name, attrs', [ + ('var_name', 'height', 'Height', + 'var_name', {'standard_name': 'height', 'long_name': 'Height'}), + (None, 'height', 'Height', + 'height', {'standard_name': 'height', 'long_name': 'Height'}), + (None, None, 'Height', + 'Height', {'long_name': 'Height'}), + (None, None, None, + 'unknown', {}), + ]) + def test_da_coord_name_from_cube(self, std_name, long_name, var_name, + name, attrs): + from iris.cube import Cube + from iris.coords import DimCoord + + latitude = DimCoord([-90, 0, 90], standard_name=std_name, + var_name=var_name, long_name=long_name) + data = [0, 0, 0] + cube = Cube(data, dim_coords_and_dims=[(latitude, 0)]) + result = xr.DataArray.from_iris(cube) + expected = xr.DataArray(data, coords=[(name, [-90, 0, 90], attrs)]) + xr.testing.assert_identical(result, expected) + + @requires_iris + def test_prevent_duplicate_coord_names(self): + from iris.cube import Cube + from iris.coords import DimCoord + + # Iris enforces unique coordinate names. Because we use a different + # name resolution order a valid iris Cube with coords that have the + # same var_name would lead to duplicate dimension names in the + # DataArray + longitude = DimCoord([0, 360], standard_name='longitude', + var_name='duplicate') + latitude = DimCoord([-90, 0, 90], standard_name='latitude', + var_name='duplicate') + data = [[0, 0, 0], [0, 0, 0]] + cube = Cube(data, dim_coords_and_dims=[(longitude, 0), (latitude, 1)]) + with pytest.raises(ValueError): + xr.DataArray.from_iris(cube) + + @requires_iris + @pytest.mark.parametrize('coord_values', [ + ['IA', 'IL', 'IN'], # non-numeric values + [0, 2, 1], # non-monotonic values + ]) + def test_fallback_to_iris_AuxCoord(self, coord_values): + from iris.cube import Cube + from iris.coords import AuxCoord + + data = [0, 0, 0] + da = xr.DataArray(data, coords=[coord_values], dims=['space']) + result = xr.DataArray.to_iris(da) + expected = Cube(data, aux_coords_and_dims=[ + (AuxCoord(coord_values, var_name='space'), 0)]) + assert result == expected From b8a342ae845a7d734d391c4b597ee689ed9c6a72 Mon Sep 17 00:00:00 2001 From: seth-p Date: Fri, 20 Jul 2018 12:04:50 -0400 Subject: [PATCH 171/282] ENH: format_array_flat() always displays first and last items. (#2293) --- doc/whats-new.rst | 6 +++ xarray/core/formatting.py | 71 ++++++++++++++++++++------- xarray/core/pycompat.py | 5 +- xarray/tests/test_dataset.py | 8 ++-- xarray/tests/test_formatting.py | 85 ++++++++++++++++++++++++++------- 5 files changed, 134 insertions(+), 41 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e392e3ce869..af485015094 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,12 @@ Documentation Enhancements ~~~~~~~~~~~~ +- DataArray coordinates and Dataset coordinates and data variables are + now displayed as `a b ... y z` rather than `a b c d ...`. + (:issue:`1186`) + By `Seth P `_. + + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/formatting.py b/xarray/core/formatting.py index 2009df3b2d1..65f3c91ca26 100644 --- a/xarray/core/formatting.py +++ b/xarray/core/formatting.py @@ -14,7 +14,9 @@ import pandas as pd from .options import OPTIONS -from .pycompat import PY2, bytes_type, dask_array_type, unicode_type +from .pycompat import ( + PY2, bytes_type, dask_array_type, unicode_type, zip_longest, +) try: from pandas.errors import OutOfBoundsDatetime @@ -64,13 +66,13 @@ def __repr__(self): return ensure_valid_repr(self.__unicode__()) -def _get_indexer_at_least_n_items(shape, n_desired): +def _get_indexer_at_least_n_items(shape, n_desired, from_end): assert 0 < n_desired <= np.prod(shape) cum_items = np.cumprod(shape[::-1]) n_steps = np.argmax(cum_items >= n_desired) stop = int(np.ceil(float(n_desired) / np.r_[1, cum_items][n_steps])) - indexer = ((0,) * (len(shape) - 1 - n_steps) + - (slice(stop),) + + indexer = (((-1 if from_end else 0),) * (len(shape) - 1 - n_steps) + + ((slice(-stop, None) if from_end else slice(stop)),) + (slice(None),) * n_steps) return indexer @@ -89,11 +91,28 @@ def first_n_items(array, n_desired): return [] if n_desired < array.size: - indexer = _get_indexer_at_least_n_items(array.shape, n_desired) + indexer = _get_indexer_at_least_n_items(array.shape, n_desired, + from_end=False) array = array[indexer] return np.asarray(array).flat[:n_desired] +def last_n_items(array, n_desired): + """Returns the last n_desired items of an array""" + # Unfortunately, we can't just do array.flat[-n_desired:] here because it + # might not be a numpy.ndarray. Moreover, access to elements of the array + # could be very expensive (e.g. if it's only available over DAP), so go out + # of our way to get them in a single call to __getitem__ using only slices. + if (n_desired == 0) or (array.size == 0): + return [] + + if n_desired < array.size: + indexer = _get_indexer_at_least_n_items(array.shape, n_desired, + from_end=True) + array = array[indexer] + return np.asarray(array).flat[-n_desired:] + + def last_item(array): """Returns the last item of an array in a list or an empty list.""" if array.size == 0: @@ -180,20 +199,36 @@ def format_array_flat(array, max_width): array that will fit within max_width characters. """ # every item will take up at least two characters, but we always want to - # print at least one item - max_possibly_relevant = max(int(np.ceil(max_width / 2.0)), 1) - relevant_items = first_n_items(array, max_possibly_relevant) - pprint_items = format_items(relevant_items) - - cum_len = np.cumsum([len(s) + 1 for s in pprint_items]) - 1 - if (max_possibly_relevant < array.size or (cum_len > max_width).any()): - end_padding = u' ...' - count = max(np.argmax((cum_len + len(end_padding)) > max_width), 1) - pprint_items = pprint_items[:count] + # print at least first and last items + max_possibly_relevant = min(max(array.size, 1), + max(int(np.ceil(max_width / 2.)), 2)) + relevant_front_items = format_items( + first_n_items(array, (max_possibly_relevant + 1) // 2)) + relevant_back_items = format_items( + last_n_items(array, max_possibly_relevant // 2)) + # interleave relevant front and back items: + # [a, b, c] and [y, z] -> [a, z, b, y, c] + relevant_items = sum(zip_longest(relevant_front_items, + reversed(relevant_back_items)), + ())[:max_possibly_relevant] + + cum_len = np.cumsum([len(s) + 1 for s in relevant_items]) - 1 + if (array.size > 2) and ((max_possibly_relevant < array.size) or + (cum_len > max_width).any()): + padding = u' ... ' + count = min(array.size, + max(np.argmax(cum_len + len(padding) - 1 > max_width), 2)) else: - end_padding = u'' - - pprint_str = u' '.join(pprint_items) + end_padding + count = array.size + padding = u'' if (count <= 1) else u' ' + + num_front = (count + 1) // 2 + num_back = count - num_front + # note that num_back is 0 <--> array.size is 0 or 1 + # <--> relevant_back_items is [] + pprint_str = (u' '.join(relevant_front_items[:num_front]) + + padding + + u' '.join(relevant_back_items[-num_back:])) return pprint_str diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index df7781ca9c1..78c26f1e92f 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -23,6 +23,7 @@ def itervalues(d): range = range zip = zip + from itertools import zip_longest from functools import reduce import builtins from urllib.request import urlretrieve @@ -41,7 +42,9 @@ def itervalues(d): return d.itervalues() range = xrange - from itertools import izip as zip, imap as map + from itertools import ( + izip as zip, imap as map, izip_longest as zip_longest, + ) reduce = reduce import __builtin__ as builtins from urllib import urlretrieve diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 4aa99b8ee5a..5c9131d0713 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -93,15 +93,15 @@ def test_repr(self): Dimensions: (dim1: 8, dim2: 9, dim3: 10, time: 20) Coordinates: - * time (time) datetime64[ns] 2000-01-01 2000-01-02 2000-01-03 ... + * time (time) datetime64[ns] 2000-01-01 2000-01-02 ... 2000-01-20 * dim2 (dim2) float64 0.0 0.5 1.0 1.5 2.0 2.5 3.0 3.5 4.0 * dim3 (dim3) %s 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' numbers (dim3) int64 0 1 2 0 0 1 1 2 2 3 Dimensions without coordinates: dim1 Data variables: - var1 (dim1, dim2) float64 -1.086 0.9973 0.283 -1.506 -0.5786 1.651 ... - var2 (dim1, dim2) float64 1.162 -1.097 -2.123 1.04 -0.4034 -0.126 ... - var3 (dim3, dim1) float64 0.5565 -0.2121 0.4563 1.545 -0.2397 0.1433 ... + var1 (dim1, dim2) float64 -1.086 0.9973 0.283 ... 0.1995 0.4684 -0.8312 + var2 (dim1, dim2) float64 1.162 -1.097 -2.123 ... 0.1302 1.267 0.3328 + var3 (dim3, dim1) float64 0.5565 -0.2121 0.4563 ... -0.2452 -0.3616 Attributes: foo: bar""") % data['dim3'].dtype # noqa: E501 actual = '\n'.join(x.rstrip() for x in repr(data).split('\n')) diff --git a/xarray/tests/test_formatting.py b/xarray/tests/test_formatting.py index 34552891778..8a1003f1ced 100644 --- a/xarray/tests/test_formatting.py +++ b/xarray/tests/test_formatting.py @@ -14,20 +14,31 @@ class TestFormatting(TestCase): def test_get_indexer_at_least_n_items(self): cases = [ - ((20,), (slice(10),)), - ((3, 20,), (0, slice(10))), - ((2, 10,), (0, slice(10))), - ((2, 5,), (slice(2), slice(None))), - ((1, 2, 5,), (0, slice(2), slice(None))), - ((2, 3, 5,), (0, slice(2), slice(None))), - ((1, 10, 1,), (0, slice(10), slice(None))), - ((2, 5, 1,), (slice(2), slice(None), slice(None))), - ((2, 5, 3,), (0, slice(4), slice(None))), - ((2, 3, 3,), (slice(2), slice(None), slice(None))), + ((20,), (slice(10),), (slice(-10, None),)), + ((3, 20,), (0, slice(10)), (-1, slice(-10, None))), + ((2, 10,), (0, slice(10)), (-1, slice(-10, None))), + ((2, 5,), (slice(2), slice(None)), + (slice(-2, None), slice(None))), + ((1, 2, 5,), (0, slice(2), slice(None)), + (-1, slice(-2, None), slice(None))), + ((2, 3, 5,), (0, slice(2), slice(None)), + (-1, slice(-2, None), slice(None))), + ((1, 10, 1,), (0, slice(10), slice(None)), + (-1, slice(-10, None), slice(None))), + ((2, 5, 1,), (slice(2), slice(None), slice(None)), + (slice(-2, None), slice(None), slice(None))), + ((2, 5, 3,), (0, slice(4), slice(None)), + (-1, slice(-4, None), slice(None))), + ((2, 3, 3,), (slice(2), slice(None), slice(None)), + (slice(-2, None), slice(None), slice(None))), ] - for shape, expected in cases: - actual = formatting._get_indexer_at_least_n_items(shape, 10) - assert expected == actual + for shape, start_expected, end_expected in cases: + actual = formatting._get_indexer_at_least_n_items(shape, 10, + from_end=False) + assert start_expected == actual + actual = formatting._get_indexer_at_least_n_items(shape, 10, + from_end=True) + assert end_expected == actual def test_first_n_items(self): array = np.arange(100).reshape(10, 5, 2) @@ -39,6 +50,16 @@ def test_first_n_items(self): with raises_regex(ValueError, 'at least one item'): formatting.first_n_items(array, 0) + def test_last_n_items(self): + array = np.arange(100).reshape(10, 5, 2) + for n in [3, 10, 13, 100, 200]: + actual = formatting.last_n_items(array, n) + expected = array.flat[-n:] + self.assertItemsEqual(expected, actual) + + with raises_regex(ValueError, 'at least one item'): + formatting.first_n_items(array, 0) + def test_last_item(self): array = np.arange(100) @@ -87,16 +108,32 @@ def test_format_items(self): assert expected == actual def test_format_array_flat(self): + actual = formatting.format_array_flat(np.arange(100), 2) + expected = '0 ... 99' + assert expected == actual + + actual = formatting.format_array_flat(np.arange(100), 9) + expected = '0 ... 99' + assert expected == actual + + actual = formatting.format_array_flat(np.arange(100), 10) + expected = '0 1 ... 99' + assert expected == actual + actual = formatting.format_array_flat(np.arange(100), 13) - expected = '0 1 2 3 4 ...' + expected = '0 1 ... 98 99' + assert expected == actual + + actual = formatting.format_array_flat(np.arange(100), 15) + expected = '0 1 2 ... 98 99' assert expected == actual actual = formatting.format_array_flat(np.arange(100.0), 11) - expected = '0.0 1.0 ...' + expected = '0.0 ... 99.0' assert expected == actual actual = formatting.format_array_flat(np.arange(100.0), 1) - expected = '0.0 ...' + expected = '0.0 ... 99.0' assert expected == actual actual = formatting.format_array_flat(np.arange(3), 5) @@ -104,11 +141,23 @@ def test_format_array_flat(self): assert expected == actual actual = formatting.format_array_flat(np.arange(4.0), 11) - expected = '0.0 1.0 ...' + expected = '0.0 ... 3.0' + assert expected == actual + + actual = formatting.format_array_flat(np.arange(0), 0) + expected = '' + assert expected == actual + + actual = formatting.format_array_flat(np.arange(1), 0) + expected = '0' + assert expected == actual + + actual = formatting.format_array_flat(np.arange(2), 0) + expected = '0 1' assert expected == actual actual = formatting.format_array_flat(np.arange(4), 0) - expected = '0 ...' + expected = '0 ... 3' assert expected == actual def test_pretty_print(self): From 46732461d01fc1a2dd097599bb384b7306f1c0a5 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Mon, 23 Jul 2018 18:33:07 +0200 Subject: [PATCH 172/282] Rename "Recipes" to "Gallery" (#2303) * Rename "Recipes" to "Gallery" * Don't use "gallery" as internal link name --- doc/gallery/README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gallery/README.txt b/doc/gallery/README.txt index 242c4f7dc91..b17f803696b 100644 --- a/doc/gallery/README.txt +++ b/doc/gallery/README.txt @@ -1,5 +1,5 @@ .. _recipes: -Recipes +Gallery ======= From 9802d618251eae88474f680999db4c23967c8f7a Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Fri, 27 Jul 2018 10:20:42 +0200 Subject: [PATCH 173/282] Make RTD builds faster (#2310) * Try to pin more packages on RTD * Unpin everything * Try default rasterio * Repin everything --- doc/conf.py | 3 ++- doc/environment.yml | 37 ++++++++++++++++++------------------- readthedocs.yml | 7 +++++-- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 5fd3bece3bd..897c0443054 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -25,7 +25,8 @@ print("python exec:", sys.executable) print("sys.path:", sys.path) for name in ('numpy scipy pandas matplotlib dask IPython seaborn ' - 'cartopy netCDF4 rasterio zarr').split(): + 'cartopy netCDF4 rasterio zarr iris flake8 ' + 'sphinx_gallery cftime').split(): try: module = importlib.import_module(name) if name == 'matplotlib': diff --git a/doc/environment.yml b/doc/environment.yml index a7683ff1824..bd134a7656f 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -1,24 +1,23 @@ name: xarray-docs channels: - conda-forge - - defaults dependencies: - python=3.6 - - numpy=1.13 - - pandas=0.21.0 - - scipy=1.0 - - bottleneck - - numpydoc=0.7.0 - - matplotlib=2.1.2 - - seaborn=0.8 - - dask=0.16.0 - - ipython=6.2.1 - - sphinx=1.5 - - netCDF4=1.3.1 - - cartopy=0.15.1 - - rasterio=0.36.0 - - sphinx-gallery - - zarr - - iris - - flake8 - - cftime + - numpy=1.14.5 + - pandas=0.23.3 + - scipy=1.1.0 + - matplotlib=2.2.2 + - seaborn=0.9.0 + - dask=0.18.2 + - ipython=6.4.0 + - netCDF4=1.4.0 + - cartopy=0.16.0 + - rasterio=1.0.1 + - zarr=2.2.0 + - iris=2.1.0 + - flake8=3.5.0 + - cftime=1.0.0 + - bottleneck=1.2 + - sphinx=1.7.6 + - numpydoc=0.8.0 + - sphinx-gallery=0.2.0 diff --git a/readthedocs.yml b/readthedocs.yml index 0129abe15aa..8e9c09c9414 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,5 +1,8 @@ +build: + image: latest conda: file: doc/environment.yml python: - version: 3 - setup_py_install: true + version: 3.6 + setup_py_install: true +formats: [] From f537ba65e9e5ca49d2dc1d9af6284e5926e0fa32 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 27 Jul 2018 16:50:54 -0700 Subject: [PATCH 174/282] DOC: add initial draft of a development roadmap for xarray (#2309) * DOC: add initial draft of a development roadmap for xarray * fix bullets --- doc/index.rst | 2 + doc/roadmap.rst | 225 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 doc/roadmap.rst diff --git a/doc/index.rst b/doc/index.rst index 7528f3cb1fa..6cd942f94a8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -74,6 +74,7 @@ Documentation * :doc:`whats-new` * :doc:`api` * :doc:`internals` +* :doc:`roadmap` * :doc:`contributing` .. toctree:: @@ -84,6 +85,7 @@ Documentation whats-new api internals + roadmap contributing See also diff --git a/doc/roadmap.rst b/doc/roadmap.rst new file mode 100644 index 00000000000..2708cb7cf8f --- /dev/null +++ b/doc/roadmap.rst @@ -0,0 +1,225 @@ +Development roadmap +=================== + +Authors: Stephan Hoyer, Joe Hamman and xarray developers + +Date: July 24, 2018 + +Xarray is an open source Python library for labeled multidimensional +arrays and datasets. + +Our philosophy +-------------- + +Why has xarray been successful? In our opinion: + +- Xarray does a great job of solving **specific use-cases** for + multidimensional data analysis: + + - The dominant use-case for xarray is for analysis of gridded + dataset in the geosciences, e.g., as part of the + `Pangeo `__ project. + - Xarray is also used more broadly in the physical sciences, where + we've found the needs for analyzing multidimensional datasets are + remarkably consistent (e.g., see + `SunPy `__ and + `PlasmaPy `__). + - Finally, xarray is used in a variety of other domains, including + finance, `probabilistic + programming `__ and + genomics. + +- Xarray is also a **domain agnostic** solution: + + - We focus on providing a flexible set of functionality related + labeled multidimensional arrays, rather than solving particular + problems. + - This facilitates collaboration between users with different needs, + and helps us attract a broad community of contributers. + - Importantly, this retains flexibility, for use cases that don't + fit particularly well into existing frameworks. + +- Xarray **integrates well** with other libraries in the scientific + Python stack. + + - We leverage first-class external libraries for core features of + xarray (e.g., NumPy for ndarrays, pandas for indexing, dask for + parallel computing) + - We expose our internal abstractions to users (e.g., + ``apply_ufunc()``), which facilitates extending xarray in various + ways. + +Together, these features have made xarray a first-class choice for +labeled multidimensional arrays in Python. + +We want to double-down on xarray's strengths by making it an even more +flexible and powerful tool for multidimensional data analysis. We want +to continue to engage xarray's core geoscience users, and to also reach +out to new domains to learn from other successful data models like those +of `yt `__ or the `OLAP +cube `__. + +Specific needs +-------------- + +The user community has voiced a number specific needs related to how +xarray interfaces with domain specific problems. Xarray may not solve +all of these issues directly, but these areas provide opportunities for +xarray to provide better, more extensible, interfaces. Some examples of +these common needs are: + +- Non-regular grids (e.g., staggered and unstructured meshes). +- Physical units. +- Lazily computed arrays (e.g., for coordinate systems). +- New file-formats. + +Technical vision +---------------- + +We think the right approach to extending xarray's user community and the +usefulness of the project is to focus on improving key interfaces that +can be used externally to meet domain-specific needs. + +We can generalize the community's needs into three main catagories: + +- More flexible grids/indexing. +- More flexible arrays/computing. +- More flexible storage backends. + +Each of these are detailed further in the subsections below. + +Flexible indexes +~~~~~~~~~~~~~~~~ + +Xarray currently keeps track of indexes associated with coordinates by +storing them in the form of a ``pandas.Index`` in special +``xarray.IndexVariable`` objects. + +The limitations of this model became clear with the addition of +``pandas.MultiIndex`` support in xarray 0.9, where a single index +corresponds to multiple xarray variables. MultiIndex support is highly +useful, but xarray now has numerous special cases to check for +MultiIndex levels. + +A cleaner model would be to elevate ``indexes`` to an explicit part of +xarray's data model, e.g., as attributes on the ``Dataset`` and +``DataArray`` classes. Indexes would need to be propagated along with +coordinates in xarray operations, but will no longer would need to have +a one-to-one correspondance with coordinate variables. Instead, an index +should be able to refer to multiple (possibly multidimensional) +coordinates that define it. See `GH +1603 `__ for full details + +Specific tasks: + +- Add an ``indexes`` attribute to ``xarray.Dataset`` and + ``xarray.Dataset``, as dictionaries that map from coordinate names to + xarray index objects. +- Use the new index interface to write wrappers for ``pandas.Index``, + ``pandas.MultiIndex`` and ``scipy.spatial.KDTree``. +- Expose the interface externally to allow third-party libraries to + implement custom indexing routines, e.g., for geospatial look-ups on + the surface of the Earth. + +In addition to the new features it directly enables, this clean up will +allow xarray to more easily implement some long-awaited features that +build upon indexing, such as groupby operations with multiple variables. + +Flexible arrays +~~~~~~~~~~~~~~~ + +Xarray currently supports wrapping multidimensional arrays defined by +NumPy, dask and to a limited-extent pandas. It would be nice to have +interfaces that allow xarray to wrap alternative N-D array +implementations, e.g.: + +- Arrays holding physical units. +- Lazily computed arrays. +- Other ndarray objects, e.g., sparse, xnd, xtensor. + +Our strategy has been to pursue upstream improvements in NumPy (see +`NEP-22 `__) +for supporting a complete duck-typing interface using with NumPy's +higher level array API. Improvements in NumPy's support for custom data +types would also be highly useful for xarray users. + +By pursuing these improvements in NumPy we hope to extend the benefits +to the full scientific Python community, and avoid tight coupling +between xarray and specific third-party libraries (e.g., for +implementing untis). This will allow xarray to maintain its domain +agnostic strengths. + +We expect that we may eventually add some minimal interfaces in xarray +for features that we delegate to external array libraries (e.g., for +getting units and changing units). If we do add these features, we +expect them to be thin wrappers, with core functionality implemented by +third-party libraries. + +Flexible storage +~~~~~~~~~~~~~~~~ + +The xarray backends module has grown in size and complexity. Much of +this growth has been "organic" and mostly to support incremental +additions to the supported backends. This has left us with a fragile +internal API that is difficult for even experienced xarray developers to +use. Moreover, the lack of a public facing API for building xarray +backends means that users can not easily build backend interface for +xarray in third-party libraries. + +The idea of refactoring the backends API and exposing it to users was +originally proposed in `GH +1970 `__. The idea would +be to develop a well tested and generic backend base class and +associated utilities for external use. Specific tasks for this +development would include: + +- Exposing an abstract backend for writing new storage systems. +- Exposing utilities for features like automatic closing of files, + LRU-caching and explicit/lazy indexing. +- Possibly moving some infrequently used backends to third-party + packages. + +Engaging more users +------------------- + +Like many open-source projects, the documentation of xarray has grown +together with the library's features. While we think that the xarray +documentation is comprehensive already, we aknowledge that the adoption +of xarray might be slowed down because of the substantial time +investment required to learn its working principles. In particular, +non-computer scientists or users less familiar with the pydata ecosystem +might find it difficult to learn xarray and realize how xarray can help +them in their daily work. + +In order to lower this adoption barrier, we propose to: + +- Develop entry-level tutorials for users with different backgrounds. For + example, we would like to develop tutorials for users with or without + previous knowledge of pandas, numpy, netCDF, etc. These tutorials may be + built as part of xarray's documentation or included in a seperate repository + to enable interactive use (e.g. mybinder.org). +- Document typical user workflows in a dedicated website, following the example + of `dask-stories + `__. +- Write a basic glossary that defines terms that might not be familiar to all + (e.g. "lazy", "labeled", "serialization", "indexing", "backend"). + +Administrative +-------------- + +Current core developers +~~~~~~~~~~~~~~~~~~~~~~~ + +- Stephan Hoyer +- Ryan Abernathey +- Joe Hamman +- Benoit Bovy +- Fabien Maussion +- Keisuke Fujii +- Maximilian Roos + +NumFOCUS +~~~~~~~~ + +On July 16, 2018, Joe and Stephan submitted xarray's fiscal sponsorship +application to NumFOCUS. From 600b34587767e06fd2092f304f5e1aba63e28f68 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 27 Jul 2018 23:07:54 -0700 Subject: [PATCH 175/282] Test against dask master and xfail current failure (#2324) --- .travis.yml | 2 -- ci/requirements-py36-dask-dev.yml | 6 +++++- xarray/tests/test_variable.py | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6df70e92954..223f1df341d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,8 +59,6 @@ matrix: - libhdf5-serial-dev - netcdf-bin - libnetcdf-dev - - python: 3.6 - env: CONDA_ENV=py36-dask-dev - python: 3.6 env: CONDA_ENV=py36-pandas-dev - python: 3.6 diff --git a/ci/requirements-py36-dask-dev.yml b/ci/requirements-py36-dask-dev.yml index 54cdb54e8fc..e580aaf3889 100644 --- a/ci/requirements-py36-dask-dev.yml +++ b/ci/requirements-py36-dask-dev.yml @@ -12,9 +12,13 @@ dependencies: - flake8 - numpy - pandas - - seaborn - scipy + - seaborn - toolz + - rasterio + - bottleneck + - zarr + - pseudonetcdf>=3.0.1 - pip: - coveralls - pytest-cov diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 290c7a6e308..cdb578aff6c 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1665,6 +1665,12 @@ def test_getitem_fancy(self): def test_getitem_1d_fancy(self): super(TestVariableWithDask, self).test_getitem_1d_fancy() + def test_equals_all_dtypes(self): + import dask + if '0.18.2' <= LooseVersion(dask.__version__) < '0.18.3': + pytest.xfail('https://github.com/pydata/xarray/issues/2318') + super(TestVariableWithDask, self).test_equals_all_dtypes() + def test_getitem_with_mask_nd_indexer(self): import dask.array as da v = Variable(['x'], da.arange(3, chunks=3)) From 2fa9dded34e06104379ad1a12c6967913998889b Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Sat, 28 Jul 2018 02:17:21 -0400 Subject: [PATCH 176/282] DOC: add interp example (#2312) * add interp example * add datetime64 example * better example of updating DataArray with interp --- doc/interpolation.rst | 17 +++++++++++++++++ xarray/core/dataarray.py | 9 +++++++++ 2 files changed, 26 insertions(+) diff --git a/doc/interpolation.rst b/doc/interpolation.rst index cd1c078fb2d..e5acbb76854 100644 --- a/doc/interpolation.rst +++ b/doc/interpolation.rst @@ -48,6 +48,23 @@ array-like, which gives the interpolated result as an array. # interpolation da.interp(time=[2.5, 3.5]) + +To interpolate data with a :py:func:`numpy.datetime64` coordinate you have to +wrap it explicitly in a :py:func:`numpy.datetime64` object. + +.. ipython:: python + + da_dt64 = xr.DataArray([1, 3], + [('time', pd.date_range('1/1/2000', '1/3/2000', periods=2))]) + da_dt64.interp(time=np.datetime64('2000-01-02')) + +The interpolated data can be merged into the original :py:class:`~xarray.DataArray` +by specifing the time periods required. + +.. ipython:: python + + da_dt64.interp(time=pd.date_range('1/1/2000', '1/3/2000', periods=3)) + .. note:: Currently, our interpolation only works for regular grids. diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 35def72c64a..710da29e7be 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -941,6 +941,15 @@ def interp(self, coords=None, method='linear', assume_sorted=False, -------- scipy.interpolate.interp1d scipy.interpolate.interpn + + Examples + ------- + >>> da = xr.DataArray([1, 3], [('x', np.arange(2))]) + >>> da.interp(x=0.5) + + array(2.0) + Coordinates: + x float64 0.5 """ if self.dtype.kind not in 'uifc': raise TypeError('interp only works for a numeric type array. ' From ded0a684136540962bcc409e6272b1cebb5af30a Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sat, 28 Jul 2018 18:05:49 +0800 Subject: [PATCH 177/282] fix doc build error after #2312 (#2326) --- doc/interpolation.rst | 4 ++-- xarray/core/dataarray.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/interpolation.rst b/doc/interpolation.rst index e5acbb76854..7b483ed03bd 100644 --- a/doc/interpolation.rst +++ b/doc/interpolation.rst @@ -49,7 +49,7 @@ array-like, which gives the interpolated result as an array. da.interp(time=[2.5, 3.5]) -To interpolate data with a :py:func:`numpy.datetime64` coordinate you have to +To interpolate data with a :py:func:`numpy.datetime64` coordinate you have to wrap it explicitly in a :py:func:`numpy.datetime64` object. .. ipython:: python @@ -58,7 +58,7 @@ wrap it explicitly in a :py:func:`numpy.datetime64` object. [('time', pd.date_range('1/1/2000', '1/3/2000', periods=2))]) da_dt64.interp(time=np.datetime64('2000-01-02')) -The interpolated data can be merged into the original :py:class:`~xarray.DataArray` +The interpolated data can be merged into the original :py:class:`~xarray.DataArray` by specifing the time periods required. .. ipython:: python diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 710da29e7be..f215bc47df8 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -943,7 +943,7 @@ def interp(self, coords=None, method='linear', assume_sorted=False, scipy.interpolate.interpn Examples - ------- + -------- >>> da = xr.DataArray([1, 3], [('x', np.arange(2))]) >>> da.interp(x=0.5) From f281945fcacbfc1dbaa48fb15546fae5317bdda8 Mon Sep 17 00:00:00 2001 From: tv3141 Date: Sat, 28 Jul 2018 17:37:13 +0100 Subject: [PATCH 178/282] Add libraries to show_versions (#2321) * Add dependencies to show_versions(). * DOC: Add libraries to dependency list. * Undo adding of h5py to docs. --- doc/installing.rst | 4 ++++ xarray/util/print_versions.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/doc/installing.rst b/doc/installing.rst index cdb1da44ef6..85cd5a02568 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -31,6 +31,10 @@ For netCDF and IO - `PseudoNetCDF `__: recommended for accessing CAMx, GEOS-Chem (bpch), NOAA ARL files, ICARTT files (ffi1001) and many other. +- `rasterio `__: for reading GeoTiffs and + other gridded raster datasets. +- `iris `__: for conversion to and from iris' + Cube objects. For accelerating xarray ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/xarray/util/print_versions.py b/xarray/util/print_versions.py index 478b867b0af..18ce40a6fae 100755 --- a/xarray/util/print_versions.py +++ b/xarray/util/print_versions.py @@ -79,6 +79,10 @@ def show_versions(as_json=False): ("h5py", lambda mod: mod.__version__), ("Nio", lambda mod: mod.__version__), ("zarr", lambda mod: mod.__version__), + ("cftime", lambda mod: mod.__version__), + ("PseudonetCDF", lambda mod: mod.__version__), + ("rasterio", lambda mod: mod.__version__), + ("iris", lambda mod: mod.__version__), ("bottleneck", lambda mod: mod.__version__), ("cyordereddict", lambda mod: mod.__version__), ("dask", lambda mod: mod.__version__), From 1ecb6e86cbd66ecb5cf3e4f74d1220722b0dc70e Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sat, 28 Jul 2018 23:09:40 -0700 Subject: [PATCH 179/282] interp() now accepts date strings as desired co-ordinate locations (#2325) * interp() now accepts date strings as desired co-ordinate locations Fixes #2284 * Consolidated tests. * Update condition for python 2 * Add additional test. * Add decorator. --- doc/interpolation.rst | 6 ++---- doc/whats-new.rst | 4 ++++ xarray/core/dataset.py | 8 ++++++++ xarray/tests/test_interp.py | 36 ++++++++++++++++++++++++------------ 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/doc/interpolation.rst b/doc/interpolation.rst index 7b483ed03bd..e5230e95dae 100644 --- a/doc/interpolation.rst +++ b/doc/interpolation.rst @@ -48,15 +48,13 @@ array-like, which gives the interpolated result as an array. # interpolation da.interp(time=[2.5, 3.5]) - -To interpolate data with a :py:func:`numpy.datetime64` coordinate you have to -wrap it explicitly in a :py:func:`numpy.datetime64` object. +To interpolate data with a :py:func:`numpy.datetime64` coordinate you can pass a string. .. ipython:: python da_dt64 = xr.DataArray([1, 3], [('time', pd.date_range('1/1/2000', '1/3/2000', periods=2))]) - da_dt64.interp(time=np.datetime64('2000-01-02')) + da_dt64.interp(time='2000-01-02') The interpolated data can be merged into the original :py:class:`~xarray.DataArray` by specifing the time periods required. diff --git a/doc/whats-new.rst b/doc/whats-new.rst index af485015094..02ab7780325 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -41,6 +41,10 @@ Enhancements (:issue:`1186`) By `Seth P `_. +- When interpolating over a ``datetime64`` axis, you can now provide a datetime string instead of a ``datetime64`` object. E.g. ``da.interp(time='1991-02-01')`` + (:issue:`2284`) + By `Deepak Cherian `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 8e039572237..4b52178ad0e 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1308,6 +1308,8 @@ def _validate_indexers(self, indexers): """ Here we make sure + indexer has a valid keys + indexer is in a valid data type + * string indexers are cast to datetime64 + if associated index is DatetimeIndex """ from .dataarray import DataArray @@ -1328,6 +1330,12 @@ def _validate_indexers(self, indexers): raise TypeError('cannot use a Dataset as an indexer') else: v = np.asarray(v) + + if ((v.dtype.kind == 'U' or v.dtype.kind == 'S') + and isinstance(self.coords[k].to_index(), + pd.DatetimeIndex)): + v = v.astype('datetime64[ns]') + if v.ndim == 0: v = as_variable(v) elif v.ndim == 1: diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py index 69a4644bc97..4a8f4e6eedf 100644 --- a/xarray/tests/test_interp.py +++ b/xarray/tests/test_interp.py @@ -462,19 +462,31 @@ def test_interp_like(): @requires_scipy -def test_datetime(): - da = xr.DataArray(np.random.randn(24), dims='time', +@pytest.mark.parametrize('x_new, expected', [ + (pd.date_range('2000-01-02', periods=3), [1, 2, 3]), + (np.array([np.datetime64('2000-01-01T12:00'), + np.datetime64('2000-01-02T12:00')]), [0.5, 1.5]), + (['2000-01-01T12:00', '2000-01-02T12:00'], [0.5, 1.5]), + (['2000-01-01T12:00'], 0.5), + pytest.param('2000-01-01T12:00', 0.5, marks=pytest.mark.xfail) +]) +def test_datetime(x_new, expected): + da = xr.DataArray(np.arange(24), dims='time', coords={'time': pd.date_range('2000-01-01', periods=24)}) - x_new = pd.date_range('2000-01-02', periods=3) actual = da.interp(time=x_new) - expected = da.isel(time=[1, 2, 3]) - assert_allclose(actual, expected) + expected_da = xr.DataArray(np.atleast_1d(expected), dims=['time'], + coords={'time': (np.atleast_1d(x_new) + .astype('datetime64[ns]'))}) - x_new = np.array([np.datetime64('2000-01-01T12:00'), - np.datetime64('2000-01-02T12:00')]) - actual = da.interp(time=x_new) - assert_allclose(actual.isel(time=0).drop('time'), - 0.5 * (da.isel(time=0) + da.isel(time=1))) - assert_allclose(actual.isel(time=1).drop('time'), - 0.5 * (da.isel(time=1) + da.isel(time=2))) + assert_allclose(actual, expected_da) + + +@requires_scipy +def test_datetime_single_string(): + da = xr.DataArray(np.arange(24), dims='time', + coords={'time': pd.date_range('2000-01-01', periods=24)}) + actual = da.interp(time='2000-01-01T12:00') + expected = xr.DataArray(0.5) + + assert_allclose(actual.drop('time'), expected) From 5d8670f6949fed60ab570075ae7dc67200f9ff51 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Mon, 30 Jul 2018 13:05:29 +0200 Subject: [PATCH 180/282] Remove test on rasterio rc and test for 0.36 instead (#2317) --- .travis.yml | 4 +--- ...sterio1.0alpha.yml => requirements-py36-rasterio-0.36.yml} | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) rename ci/{requirements-py36-rasterio1.0alpha.yml => requirements-py36-rasterio-0.36.yml} (86%) diff --git a/.travis.yml b/.travis.yml index 223f1df341d..951b151d829 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ matrix: - python: 3.6 env: CONDA_ENV=py36-pynio-dev - python: 3.6 - env: CONDA_ENV=py36-rasterio1.0alpha + env: CONDA_ENV=py36-rasterio-0.36 - python: 3.6 env: CONDA_ENV=py36-zarr-dev - python: 3.5 @@ -67,8 +67,6 @@ matrix: env: CONDA_ENV=py36-condaforge-rc - python: 3.6 env: CONDA_ENV=py36-pynio-dev - - python: 3.6 - env: CONDA_ENV=py36-rasterio1.0alpha - python: 3.6 env: CONDA_ENV=py36-zarr-dev diff --git a/ci/requirements-py36-rasterio1.0alpha.yml b/ci/requirements-py36-rasterio-0.36.yml similarity index 86% rename from ci/requirements-py36-rasterio1.0alpha.yml rename to ci/requirements-py36-rasterio-0.36.yml index 15ba13e753b..5c724e1b981 100644 --- a/ci/requirements-py36-rasterio1.0alpha.yml +++ b/ci/requirements-py36-rasterio-0.36.yml @@ -1,7 +1,6 @@ name: test_env channels: - conda-forge - - conda-forge/label/dev dependencies: - python=3.6 - cftime @@ -17,7 +16,7 @@ dependencies: - scipy - seaborn - toolz - - rasterio>=1.* + - rasterio=0.36.0 - bottleneck - pip: - coveralls From 603911d203ffdfa10528e838c5d98164b95f6a08 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 31 Jul 2018 15:28:43 -0700 Subject: [PATCH 181/282] Support for additional axis kwargs (#2294) 1. Support xscale, yscale, xticks, yticks, xlim, ylim kwargs. 2. Small fixes: 1. Forgot to replace autofmt_xdate for 2D plots. 2. Use matplotlib's axis inverting methods. 3. Don't automatically set histogram ylabel to be 'count'. --- doc/plotting.rst | 9 ++-- doc/whats-new.rst | 3 ++ xarray/plot/plot.py | 96 +++++++++++++++++++++++++++++++-------- xarray/tests/test_plot.py | 69 ++++++++++++++++++++++++++-- 4 files changed, 149 insertions(+), 28 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index 54fa2f57ac8..43faa83b9da 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -212,8 +212,6 @@ If required, the automatic legend can be turned off using ``add_legend=False``. ``hue`` can be passed directly to :py:func:`xarray.plot` as `air.isel(lon=10, lat=[19,21,22]).plot(hue='lat')`. - - Dimension along y-axis ~~~~~~~~~~~~~~~~~~~~~~ @@ -224,8 +222,8 @@ It is also possible to make line plots such that the data are on the x-axis and @savefig plotting_example_xy_kwarg.png air.isel(time=10, lon=[10, 11]).plot(y='lat', hue='lon') -Changing Axes Direction ------------------------ +Other axes kwargs +----------------- The keyword arguments ``xincrease`` and ``yincrease`` let you control the axes direction. @@ -234,6 +232,9 @@ The keyword arguments ``xincrease`` and ``yincrease`` let you control the axes d @savefig plotting_example_xincrease_yincrease_kwarg.png air.isel(time=10, lon=[10, 11]).plot.line(y='lat', hue='lon', xincrease=False, yincrease=False) +In addition, one can use ``xscale, yscale`` to set axes scaling; ``xticks, yticks`` to set axes ticks and ``xlim, ylim`` to set axes limits. These accept the same values as the matplotlib methods ``Axes.set_(x,y)scale()``, ``Axes.set_(x,y)ticks()``, ``Axes.set_(x,y)lim()`` respectively. + + Two Dimensions -------------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 02ab7780325..0ecdc28e98c 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,9 @@ Documentation Enhancements ~~~~~~~~~~~~ +- :py:meth:`plot()` now accepts the kwargs ``xscale, yscale, xlim, ylim, xticks, yticks`` just like Pandas. Also ``xincrease=False, yincrease=False`` now use matplotlib's axis inverting methods instead of setting limits. + By `Deepak Cherian `_. (:issue:`2224`) + - DataArray coordinates and Dataset coordinates and data variables are now displayed as `a b ... y z` rather than `a b c d ...`. (:issue:`1186`) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 2a7fb08efda..179f41e9e42 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -270,6 +270,10 @@ def line(darray, *args, **kwargs): Coordinates for x, y axis. Only one of these may be specified. The other coordinate plots values from the DataArray on which this plot method is called. + xscale, yscale : 'linear', 'symlog', 'log', 'logit', optional + Specifies scaling for the x- and y-axes respectively + xticks, yticks : Specify tick locations for x- and y-axes + xlim, ylim : Specify x- and y-axes limits xincrease : None, True, or False, optional Should the values on the x axes be increasing from left to right? if None, use the default for the matplotlib function. @@ -305,8 +309,14 @@ def line(darray, *args, **kwargs): hue = kwargs.pop('hue', None) x = kwargs.pop('x', None) y = kwargs.pop('y', None) - xincrease = kwargs.pop('xincrease', True) - yincrease = kwargs.pop('yincrease', True) + xincrease = kwargs.pop('xincrease', None) # default needs to be None + yincrease = kwargs.pop('yincrease', None) + xscale = kwargs.pop('xscale', None) # default needs to be None + yscale = kwargs.pop('yscale', None) + xticks = kwargs.pop('xticks', None) + yticks = kwargs.pop('yticks', None) + xlim = kwargs.pop('xlim', None) + ylim = kwargs.pop('ylim', None) add_legend = kwargs.pop('add_legend', True) _labels = kwargs.pop('_labels', True) if args is (): @@ -343,7 +353,8 @@ def line(darray, *args, **kwargs): xlabels.set_rotation(30) xlabels.set_ha('right') - _update_axes_limits(ax, xincrease, yincrease) + _update_axes(ax, xincrease, yincrease, xscale, yscale, + xticks, yticks, xlim, ylim) return primitive @@ -378,37 +389,69 @@ def hist(darray, figsize=None, size=None, aspect=None, ax=None, **kwargs): """ ax = get_axis(figsize, size, aspect, ax) + xincrease = kwargs.pop('xincrease', None) # default needs to be None + yincrease = kwargs.pop('yincrease', None) + xscale = kwargs.pop('xscale', None) # default needs to be None + yscale = kwargs.pop('yscale', None) + xticks = kwargs.pop('xticks', None) + yticks = kwargs.pop('yticks', None) + xlim = kwargs.pop('xlim', None) + ylim = kwargs.pop('ylim', None) + no_nan = np.ravel(darray.values) no_nan = no_nan[pd.notnull(no_nan)] primitive = ax.hist(no_nan, **kwargs) - ax.set_ylabel('Count') - ax.set_title('Histogram') ax.set_xlabel(label_from_attrs(darray)) + _update_axes(ax, xincrease, yincrease, xscale, yscale, + xticks, yticks, xlim, ylim) + return primitive -def _update_axes_limits(ax, xincrease, yincrease): +def _update_axes(ax, xincrease, yincrease, + xscale=None, yscale=None, + xticks=None, yticks=None, + xlim=None, ylim=None): """ - Update axes in place to increase or decrease - For use in _plot2d + Update axes with provided parameters """ if xincrease is None: pass - elif xincrease: - ax.set_xlim(sorted(ax.get_xlim())) - elif not xincrease: - ax.set_xlim(sorted(ax.get_xlim(), reverse=True)) + elif xincrease and ax.xaxis_inverted(): + ax.invert_xaxis() + elif not xincrease and not ax.xaxis_inverted(): + ax.invert_xaxis() if yincrease is None: pass - elif yincrease: - ax.set_ylim(sorted(ax.get_ylim())) - elif not yincrease: - ax.set_ylim(sorted(ax.get_ylim(), reverse=True)) + elif yincrease and ax.yaxis_inverted(): + ax.invert_yaxis() + elif not yincrease and not ax.yaxis_inverted(): + ax.invert_yaxis() + + # The default xscale, yscale needs to be None. + # If we set a scale it resets the axes formatters, + # This means that set_xscale('linear') on a datetime axis + # will remove the date labels. So only set the scale when explicitly + # asked to. https://github.com/matplotlib/matplotlib/issues/8740 + if xscale is not None: + ax.set_xscale(xscale) + if yscale is not None: + ax.set_yscale(yscale) + + if xticks is not None: + ax.set_xticks(xticks) + if yticks is not None: + ax.set_yticks(yticks) + + if xlim is not None: + ax.set_xlim(xlim) + if ylim is not None: + ax.set_ylim(ylim) # MUST run before any 2d plotting functions are defined since @@ -500,6 +543,10 @@ def _plot2d(plotfunc): If passed, make column faceted plots on this dimension name col_wrap : integer, optional Use together with ``col`` to wrap faceted plots + xscale, yscale : 'linear', 'symlog', 'log', 'logit', optional + Specifies scaling for the x- and y-axes respectively + xticks, yticks : Specify tick locations for x- and y-axes + xlim, ylim : Specify x- and y-axes limits xincrease : None, True, or False, optional Should the values on the x axes be increasing from left to right? if None, use the default for the matplotlib function. @@ -577,7 +624,8 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, cmap=None, center=None, robust=False, extend=None, levels=None, infer_intervals=None, colors=None, subplot_kws=None, cbar_ax=None, cbar_kwargs=None, - **kwargs): + xscale=None, yscale=None, xticks=None, yticks=None, + xlim=None, ylim=None, **kwargs): # All 2d plots in xarray share this function signature. # Method signature below should be consistent. @@ -723,11 +771,17 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, raise ValueError("cbar_ax and cbar_kwargs can't be used with " "add_colorbar=False.") - _update_axes_limits(ax, xincrease, yincrease) + _update_axes(ax, xincrease, yincrease, xscale, yscale, + xticks, yticks, xlim, ylim) # Rotate dates on xlabels + # Do this without calling autofmt_xdate so that x-axes ticks + # on other subplots (if any) are not deleted. + # https://stackoverflow.com/questions/17430105/autofmt-xdate-deletes-x-axis-labels-of-all-subplots if np.issubdtype(xval.dtype, np.datetime64): - ax.get_figure().autofmt_xdate() + for xlabels in ax.get_xticklabels(): + xlabels.set_rotation(30) + xlabels.set_ha('right') return primitive @@ -739,7 +793,9 @@ def plotmethod(_PlotMethods_obj, x=None, y=None, figsize=None, size=None, add_labels=True, vmin=None, vmax=None, cmap=None, colors=None, center=None, robust=False, extend=None, levels=None, infer_intervals=None, subplot_kws=None, - cbar_ax=None, cbar_kwargs=None, **kwargs): + cbar_ax=None, cbar_kwargs=None, + xscale=None, yscale=None, xticks=None, yticks=None, + xlim=None, ylim=None, **kwargs): """ The method should have the same signature as the function. diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 90d30946c9c..4e5ea8fc623 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -426,10 +426,6 @@ def test_xlabel_uses_name(self): self.darray.plot.hist() assert 'testpoints [testunits]' == plt.gca().get_xlabel() - def test_ylabel_is_count(self): - self.darray.plot.hist() - assert 'Count' == plt.gca().get_ylabel() - def test_title_is_histogram(self): self.darray.plot.hist() assert 'Histogram' == plt.gca().get_title() @@ -1675,3 +1671,68 @@ def test_plot_cftime_data_error(): data = DataArray(data, coords=[np.arange(5)], dims=['x']) with raises_regex(NotImplementedError, 'cftime.datetime'): data.plot() + + +test_da_list = [DataArray(easy_array((10, ))), + DataArray(easy_array((10, 3))), + DataArray(easy_array((10, 3, 2)))] + + +@requires_matplotlib +class TestAxesKwargs(object): + @pytest.mark.parametrize('da', test_da_list) + @pytest.mark.parametrize('xincrease', [True, False]) + def test_xincrease_kwarg(self, da, xincrease): + plt.clf() + da.plot(xincrease=xincrease) + assert plt.gca().xaxis_inverted() == (not xincrease) + + @pytest.mark.parametrize('da', test_da_list) + @pytest.mark.parametrize('yincrease', [True, False]) + def test_yincrease_kwarg(self, da, yincrease): + plt.clf() + da.plot(yincrease=yincrease) + assert plt.gca().yaxis_inverted() == (not yincrease) + + @pytest.mark.parametrize('da', test_da_list) + @pytest.mark.parametrize('xscale', ['linear', 'log', 'logit', 'symlog']) + def test_xscale_kwarg(self, da, xscale): + plt.clf() + da.plot(xscale=xscale) + assert plt.gca().get_xscale() == xscale + + @pytest.mark.parametrize('da', [DataArray(easy_array((10, ))), + DataArray(easy_array((10, 3)))]) + @pytest.mark.parametrize('yscale', ['linear', 'log', 'logit', 'symlog']) + def test_yscale_kwarg(self, da, yscale): + plt.clf() + da.plot(yscale=yscale) + assert plt.gca().get_yscale() == yscale + + @pytest.mark.parametrize('da', test_da_list) + def test_xlim_kwarg(self, da): + plt.clf() + expected = (0.0, 1000.0) + da.plot(xlim=[0, 1000]) + assert plt.gca().get_xlim() == expected + + @pytest.mark.parametrize('da', test_da_list) + def test_ylim_kwarg(self, da): + plt.clf() + da.plot(ylim=[0, 1000]) + expected = (0.0, 1000.0) + assert plt.gca().get_ylim() == expected + + @pytest.mark.parametrize('da', test_da_list) + def test_xticks_kwarg(self, da): + plt.clf() + da.plot(xticks=np.arange(5)) + expected = np.arange(5).tolist() + assert np.all(plt.gca().get_xticks() == expected) + + @pytest.mark.parametrize('da', test_da_list) + def test_yticks_kwarg(self, da): + plt.clf() + da.plot(yticks=np.arange(5)) + expected = np.arange(5) + assert np.all(plt.gca().get_yticks() == expected) From c86810b5ab3fc1cd47c1c0ed07e002b797b27eaf Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Wed, 1 Aug 2018 14:55:18 -0400 Subject: [PATCH 182/282] Better error message on invalid types (#2331) * better error message on invalid types * whats new --- doc/whats-new.rst | 4 ++++ xarray/core/variable.py | 3 +++ xarray/tests/test_dataset.py | 7 ++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0ecdc28e98c..e48c59d0e66 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -48,6 +48,10 @@ Enhancements (:issue:`2284`) By `Deepak Cherian `_. +- A clear error message is now displayed if a ``set`` or ``dict`` is passed in place of an array + (:issue:`2331`) + By `Maximilian Roos `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 52d470accfe..8b6a3d3bf21 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -92,6 +92,9 @@ def as_variable(obj, name=None): obj = Variable([], obj) elif isinstance(obj, (pd.Index, IndexVariable)) and obj.name is not None: obj = Variable(obj.name, obj) + elif isinstance(obj, (set, dict)): + raise TypeError( + "variable %r has invalid type %r" % (name, type(obj))) elif name is not None: data = as_compatible_data(obj) if data.ndim != 1: diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 5c9131d0713..8a21f64c94a 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -845,7 +845,7 @@ def test_isel(self): inds = np.nonzero(np.array(data[v].dims) == d)[0] for ind in inds: slice_list[ind] = s - expected = data[v].values[slice_list] + expected = data[v].values[tuple(slice_list)] actual = ret[v].values np.testing.assert_array_equal(expected, actual) @@ -4218,6 +4218,11 @@ def test_dataset_constructor_aligns_to_explicit_coords( assert_equal(expected, result) +def test_error_message_on_set_supplied(): + with pytest.raises(TypeError, message='has invalid type set'): + xr.Dataset(dict(date=[1, 2, 3], sec={4})) + + @pytest.mark.parametrize('unaligned_coords', ( {'y': ('b', np.asarray([2, 1, 0]))}, )) From 56381ef444c5e699443e8b4e08611060ad5c9507 Mon Sep 17 00:00:00 2001 From: Graham Inggs Date: Thu, 2 Aug 2018 04:18:02 +0200 Subject: [PATCH 183/282] Let test_unicode_data pass on big-endian systems too (#2333) --- xarray/tests/test_dataset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 8a21f64c94a..c0516ed7e56 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -5,6 +5,7 @@ from io import StringIO from textwrap import dedent import warnings +import sys import numpy as np import pandas as pd @@ -182,15 +183,16 @@ def test_unicode_data(self): data = Dataset({u'foø': [u'ba®']}, attrs={u'å': u'∑'}) repr(data) # should not raise + byteorder = '<' if sys.byteorder == 'little' else '>' expected = dedent(u"""\ Dimensions: (foø: 1) Coordinates: - * foø (foø) Date: Mon, 6 Aug 2018 08:46:59 +0900 Subject: [PATCH 184/282] local flake8 (#2343) --- xarray/backends/zarr.py | 1 - xarray/convert.py | 2 +- xarray/plot/utils.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index c5043ce8a47..a64ca858ed3 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1,6 +1,5 @@ from __future__ import absolute_import, division, print_function -from itertools import product from distutils.version import LooseVersion import numpy as np diff --git a/xarray/convert.py b/xarray/convert.py index bc639c68455..6cff72103ff 100644 --- a/xarray/convert.py +++ b/xarray/convert.py @@ -2,7 +2,7 @@ """ from __future__ import absolute_import, division, print_function -from collections import Counter, OrderedDict +from collections import Counter import numpy as np import pandas as pd diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 4b9645e02d5..1ddb02352be 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -3,8 +3,6 @@ import warnings import numpy as np -import pandas as pd -import pkg_resources import textwrap from ..core.pycompat import basestring From b6a5af3c16f71be18968f849e4db845e946d5d5c Mon Sep 17 00:00:00 2001 From: Tony Tung Date: Mon, 6 Aug 2018 07:57:15 -0700 Subject: [PATCH 185/282] Use sorted fixtures to ensure that test gathering is consistent. (#2339) pytest-xdist expects that test gathering produces the same results and in the same order. Because sets, prior to python 3.7, do not guarantee ordering, this can cause pytest -n (from pytest-xdist) to fail. --- doc/whats-new.rst | 1 + xarray/tests/test_coding_times.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e48c59d0e66..113585837c3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -61,6 +61,7 @@ Bug fixes attribute being set. (:issue:`2201`) By `Thomas Voigt `_. +- Tests can be run in parallel with pytest-xdist .. _whats-new.0.10.8: diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 4d6ca731bb2..e763af4984c 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -16,10 +16,12 @@ requires_cftime_or_netCDF4, has_cftime, has_dask) -_NON_STANDARD_CALENDARS = {'noleap', '365_day', '360_day', - 'julian', 'all_leap', '366_day'} -_ALL_CALENDARS = _NON_STANDARD_CALENDARS.union( - coding.times._STANDARD_CALENDARS) +_NON_STANDARD_CALENDARS_SET = {'noleap', '365_day', '360_day', + 'julian', 'all_leap', '366_day'} +_ALL_CALENDARS = sorted(_NON_STANDARD_CALENDARS_SET.union( + coding.times._STANDARD_CALENDARS)) +_NON_STANDARD_CALENDARS = sorted(_NON_STANDARD_CALENDARS_SET) +_STANDARD_CALENDARS = sorted(coding.times._STANDARD_CALENDARS) _CF_DATETIME_NUM_DATES_UNITS = [ (np.arange(10), 'days since 2000-01-01'), (np.arange(10).astype('float64'), 'days since 2000-01-01'), @@ -49,7 +51,7 @@ ] _CF_DATETIME_TESTS = [num_dates_units + (calendar,) for num_dates_units, calendar in product(_CF_DATETIME_NUM_DATES_UNITS, - coding.times._STANDARD_CALENDARS)] + _STANDARD_CALENDARS)] @np.vectorize @@ -161,7 +163,7 @@ def test_decode_cf_datetime_non_iso_strings(): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') @pytest.mark.parametrize( ['calendar', 'enable_cftimeindex'], - product(coding.times._STANDARD_CALENDARS, [False, True])) + product(_STANDARD_CALENDARS, [False, True])) def test_decode_standard_calendar_inside_timestamp_range( calendar, enable_cftimeindex): if enable_cftimeindex: @@ -263,7 +265,7 @@ def test_decode_dates_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') @pytest.mark.parametrize( ['calendar', 'enable_cftimeindex'], - product(coding.times._STANDARD_CALENDARS, [False, True])) + product(_STANDARD_CALENDARS, [False, True])) def test_decode_standard_calendar_single_element_inside_timestamp_range( calendar, enable_cftimeindex): if enable_cftimeindex: @@ -329,7 +331,7 @@ def test_decode_single_element_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') @pytest.mark.parametrize( ['calendar', 'enable_cftimeindex'], - product(coding.times._STANDARD_CALENDARS, [False, True])) + product(_STANDARD_CALENDARS, [False, True])) def test_decode_standard_calendar_multidim_time_inside_timestamp_range( calendar, enable_cftimeindex): if enable_cftimeindex: @@ -681,7 +683,7 @@ def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): ds[v].attrs['calendar'] = calendar if (not has_cftime and enable_cftimeindex and - calendar not in coding.times._STANDARD_CALENDARS): + calendar not in _STANDARD_CALENDARS): with pytest.raises(ValueError): with set_options(enable_cftimeindex=enable_cftimeindex): ds = decode_cf(ds) @@ -690,7 +692,7 @@ def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): ds = decode_cf(ds) if (enable_cftimeindex and - calendar not in coding.times._STANDARD_CALENDARS): + calendar not in _STANDARD_CALENDARS): assert ds.test.dtype == np.dtype('O') else: assert ds.test.dtype == np.dtype('M8[ns]') From 0b181226bbb1c26adfdd5d47d567fb78d0a450fa Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Tue, 7 Aug 2018 07:38:52 +0900 Subject: [PATCH 186/282] apply_ufunc now raises a ValueError when the size of input_core_dims is inconsistent with number of argument (#2342) * Fixes #2341 * commit forgotten files. * Avoid using np.gradient * if -> elif --- doc/whats-new.rst | 7 ++++++- xarray/core/computation.py | 5 +++++ xarray/tests/test_computation.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 113585837c3..fd359ee14f8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -56,13 +56,18 @@ Enhancements Bug fixes ~~~~~~~~~ -- Fixed ``DataArray.to_iris()`` failure while creating ``DimCoord`` by +- Fixed ``DataArray.to_iris()`` failure while creating ``DimCoord`` by falling back to creating ``AuxCoord``. Fixed dependency on ``var_name`` attribute being set. (:issue:`2201`) By `Thomas Voigt `_. - Tests can be run in parallel with pytest-xdist +- Now :py:func:`xr.apply_ufunc` raises a ValueError when the size of +``input_core_dims`` is inconsistent with the number of arguments. + (:issue:`2341`) + By `Keisuke Fujii `_. + .. _whats-new.0.10.8: v0.10.8 (18 July 2018) diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 1f771a4103c..bdba72cb48a 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -920,6 +920,11 @@ def earth_mover_distance(first_samples, if input_core_dims is None: input_core_dims = ((),) * (len(args)) + elif len(input_core_dims) != len(args): + raise ValueError( + 'input_core_dims must be None or a tuple with the length same to ' + 'the number of arguments. Given input_core_dims: {}, ' + 'number of args: {}.'.format(input_core_dims, len(args))) signature = _UFuncSignature(input_core_dims, output_core_dims) diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index e30e7e31390..ca8e4e59737 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -274,6 +274,22 @@ def func(x): assert_identical(expected_dataset_x, first_element(dataset.groupby('y'), 'x')) + def multiply(*args): + val = args[0] + for arg in args[1:]: + val = val * arg + return val + + # regression test for GH:2341 + with pytest.raises(ValueError): + apply_ufunc(multiply, data_array, data_array['y'].values, + input_core_dims=[['y']], output_core_dims=[['y']]) + expected = xr.DataArray(multiply(data_array, data_array['y']), + dims=['x', 'y'], coords=data_array.coords) + actual = apply_ufunc(multiply, data_array, data_array['y'].values, + input_core_dims=[['y'], []], output_core_dims=[['y']]) + assert_identical(expected, actual) + def test_apply_output_core_dimension(): From 0a60a52521e41dce897e265ac549cbf88c670faa Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Tue, 7 Aug 2018 14:05:34 -0400 Subject: [PATCH 187/282] Add xskillscore to project lists (#2350) --- doc/faq.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/faq.rst b/doc/faq.rst index 170a1e17bdc..3cc395b4e89 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -195,6 +195,7 @@ Geosciences - `xgcm `_: Extends the xarray data model to understand finite volume grid cells (common in General Circulation Models) and provides interpolation and difference operations for such grids. - `xmitgcm `_: a python package for reading `MITgcm `_ binary MDS files into xarray data structures. - `xshape `_: Tools for working with shapefiles, topographies, and polygons in xarray. +- `xskillscore `_: Metrics for verifying forecasts. Machine Learning ~~~~~~~~~~~~~~~~ From 04458670782c0b6fdba7e7021055155b2a6f284a Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Wed, 8 Aug 2018 10:14:01 +0900 Subject: [PATCH 188/282] dask.ghost -> dask.overlap (#2349) --- doc/whats-new.rst | 3 +++ xarray/core/dask_array_ops.py | 30 +++++++++++++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fd359ee14f8..3641a072e2e 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -62,6 +62,9 @@ Bug fixes (:issue:`2201`) By `Thomas Voigt `_. - Tests can be run in parallel with pytest-xdist +- Follow up the renamings in dask; from dask.ghost to dask.overlap + By `Keisuke Fujii `_. + - Now :py:func:`xr.apply_ufunc` raises a ValueError when the size of ``input_core_dims`` is inconsistent with the number of arguments. diff --git a/xarray/core/dask_array_ops.py b/xarray/core/dask_array_ops.py index 55ba1c1cbc6..423a65aa3c2 100644 --- a/xarray/core/dask_array_ops.py +++ b/xarray/core/dask_array_ops.py @@ -1,4 +1,5 @@ from __future__ import absolute_import, division, print_function +from distutils.version import LooseVersion import numpy as np @@ -6,7 +7,15 @@ from . import dtypes try: + import dask import dask.array as da + # Note: dask has used `ghost` before 0.18.2 + if LooseVersion(dask.__version__) <= LooseVersion('0.18.2'): + overlap = da.ghost.ghost + trim_internal = da.ghost.trim_internal + else: + overlap = da.overlap.overlap + trim_internal = da.overlap.trim_internal except ImportError: pass @@ -15,26 +24,25 @@ def dask_rolling_wrapper(moving_func, a, window, min_count=None, axis=-1): '''wrapper to apply bottleneck moving window funcs on dask arrays''' dtype, fill_value = dtypes.maybe_promote(a.dtype) a = a.astype(dtype) - # inputs for ghost + # inputs for overlap if axis < 0: axis = a.ndim + axis depth = {d: 0 for d in range(a.ndim)} depth[axis] = (window + 1) // 2 boundary = {d: fill_value for d in range(a.ndim)} - # create ghosted arrays - ag = da.ghost.ghost(a, depth=depth, boundary=boundary) + # Create overlap array. + ag = overlap(a, depth=depth, boundary=boundary) # apply rolling func out = ag.map_blocks(moving_func, window, min_count=min_count, axis=axis, dtype=a.dtype) # trim array - result = da.ghost.trim_internal(out, depth) + result = trim_internal(out, depth) return result def rolling_window(a, axis, window, center, fill_value): """ Dask's equivalence to np.utils.rolling_window """ orig_shape = a.shape - # inputs for ghost if axis < 0: axis = a.ndim + axis depth = {d: 0 for d in range(a.ndim)} @@ -50,7 +58,7 @@ def rolling_window(a, axis, window, center, fill_value): "more evenly divides the shape of your array." % (window, depth[axis], min(a.chunks[axis]))) - # Although dask.ghost pads values to boundaries of the array, + # Although dask.overlap pads values to boundaries of the array, # the size of the generated array is smaller than what we want # if center == False. if center: @@ -60,12 +68,12 @@ def rolling_window(a, axis, window, center, fill_value): start, end = window - 1, 0 pad_size = max(start, end) + offset - depth[axis] drop_size = 0 - # pad_size becomes more than 0 when the ghosted array is smaller than + # pad_size becomes more than 0 when the overlapped array is smaller than # needed. In this case, we need to enlarge the original array by padding - # before ghosting. + # before overlapping. if pad_size > 0: if pad_size < depth[axis]: - # Ghosting requires each chunk larger than depth. If pad_size is + # overlapping requires each chunk larger than depth. If pad_size is # smaller than the depth, we enlarge this and truncate it later. drop_size = depth[axis] - pad_size pad_size = depth[axis] @@ -78,8 +86,8 @@ def rolling_window(a, axis, window, center, fill_value): boundary = {d: fill_value for d in range(a.ndim)} - # create ghosted arrays - ag = da.ghost.ghost(a, depth=depth, boundary=boundary) + # create overlap arrays + ag = overlap(a, depth=depth, boundary=boundary) # apply rolling func def func(x, window, axis=-1): From fe99a22ca7bcb1f854c22f5f6894d3c5d40774a6 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sat, 11 Aug 2018 01:09:29 +0900 Subject: [PATCH 189/282] Mark some tests related to cdat-lite as xfail (#2354) --- xarray/tests/test_dataarray.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index fdca3fd8d13..2950e97cc75 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2915,6 +2915,7 @@ def test_to_masked_array(self): ma = da.to_masked_array() assert len(ma.mask) == N + @pytest.mark.xfail # GH:2332 TODO fix this in upstream? def test_to_and_from_cdms2_classic(self): """Classic with 1D axes""" pytest.importorskip('cdms2') @@ -2949,6 +2950,7 @@ def test_to_and_from_cdms2_classic(self): assert_array_equal(original.coords[coord_name], back.coords[coord_name]) + @pytest.mark.xfail # GH:2332 TODO fix this in upstream? def test_to_and_from_cdms2_sgrid(self): """Curvilinear (structured) grid @@ -2975,6 +2977,7 @@ def test_to_and_from_cdms2_sgrid(self): assert_array_equal(original.coords['lat'], back.coords['lat']) assert_array_equal(original.coords['lon'], back.coords['lon']) + @pytest.mark.xfail # GH:2332 TODO fix this in upstream? def test_to_and_from_cdms2_ugrid(self): """Unstructured grid""" pytest.importorskip('cdms2') From 846e28f8862b150352512f8e3d05bcb9db57a1a3 Mon Sep 17 00:00:00 2001 From: Benoit Bovy Date: Fri, 10 Aug 2018 22:13:08 +0200 Subject: [PATCH 190/282] DOC: move xarray related projects to top-level TOC section (#2357) * move xarray third-party projects to its own top-level TOC section * add hvplot --- doc/faq.rst | 63 +------------------------------------ doc/index.rst | 2 ++ doc/related-projects.rst | 68 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 62 deletions(-) create mode 100644 doc/related-projects.rst diff --git a/doc/faq.rst b/doc/faq.rst index 3cc395b4e89..9313481f50a 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -160,71 +160,10 @@ methods for converting back and forth between xarray and these libraries. See :py:meth:`~xarray.DataArray.to_iris` and :py:meth:`~xarray.DataArray.to_cdms2` for more details. -.. _faq.other_projects: - What other projects leverage xarray? ------------------------------------ -Here are several existing libraries that build functionality upon xarray. - -Geosciences -~~~~~~~~~~~ - -- `aospy `_: Automated analysis and management of gridded climate data. -- `infinite-diff `_: xarray-based finite-differencing, focused on gridded climate/meterology data -- `marc_analysis `_: Analysis package for CESM/MARC experiments and output. -- `MPAS-Analysis `_: Analysis for simulations produced with Model for Prediction Across Scales (MPAS) components and the Accelerated Climate Model for Energy (ACME). -- `OGGM `_: Open Global Glacier Model -- `Oocgcm `_: Analysis of large gridded geophysical datasets -- `Open Data Cube `_: Analysis toolkit of continental scale Earth Observation data from satellites. -- `Pangaea: `_: xarray extension for gridded land surface & weather model output). -- `Pangeo `_: A community effort for big data geoscience in the cloud. -- `PyGDX `_: Python 3 package for - accessing data stored in GAMS Data eXchange (GDX) files. Also uses a custom - subclass. -- `Regionmask `_: plotting and creation of masks of spatial regions -- `salem `_: Adds geolocalised subsetting, masking, and plotting operations to xarray's data structures via accessors. -- `Spyfit `_: FTIR spectroscopy of the atmosphere -- `windspharm `_: Spherical - harmonic wind analysis in Python. -- `wrf-python `_: A collection of diagnostic and interpolation routines for use with output of the Weather Research and Forecasting (WRF-ARW) Model. -- `xarray-simlab `_: xarray extension for computer model simulations. -- `xarray-topo `_: xarray extension for topographic analysis and modelling. -- `xbpch `_: xarray interface for bpch files. -- `xESMF `_: Universal Regridder for Geospatial Data. -- `xgcm `_: Extends the xarray data model to understand finite volume grid cells (common in General Circulation Models) and provides interpolation and difference operations for such grids. -- `xmitgcm `_: a python package for reading `MITgcm `_ binary MDS files into xarray data structures. -- `xshape `_: Tools for working with shapefiles, topographies, and polygons in xarray. -- `xskillscore `_: Metrics for verifying forecasts. - -Machine Learning -~~~~~~~~~~~~~~~~ -- `cesium `_: machine learning for time series analysis -- `Elm `_: Parallel machine learning on xarray data structures -- `sklearn-xarray (1) `_: Combines scikit-learn and xarray (1). -- `sklearn-xarray (2) `_: Combines scikit-learn and xarray (2). - -Extend xarray capabilities -~~~~~~~~~~~~~~~~~~~~~~~~~~ -- `Collocate `_: Collocate xarray trajectories in arbitrary physical dimensions -- `eofs `_: EOF analysis in Python. -- `xarray_extras `_: Advanced algorithms for xarray objects (e.g. intergrations/interpolations). -- `xrft `_: Fourier transforms for xarray data. -- `xr-scipy `_: A lightweight scipy wrapper for xarray. -- `X-regression `_: Multiple linear regression from Statsmodels library coupled with Xarray library. -- `xyzpy `_: Easily generate high dimensional data, including parallelization. - -Visualization -~~~~~~~~~~~~~ -- `Datashader `_, `geoviews `_, `holoviews `_, : visualization packages for large data -- `psyplot `_: Interactive data visualization with python. - -Other -~~~~~ -- `ptsa `_: EEG Time Series Analysis -- `pycalphad `_: Computational Thermodynamics in Python - -More projects can be found at the `"xarray" Github topic `_. +See section :ref:`related-projects`. How should I cite xarray? ------------------------- diff --git a/doc/index.rst b/doc/index.rst index 6cd942f94a8..e66c448f780 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -76,6 +76,7 @@ Documentation * :doc:`internals` * :doc:`roadmap` * :doc:`contributing` +* :doc:`related-projects` .. toctree:: :maxdepth: 1 @@ -87,6 +88,7 @@ Documentation internals roadmap contributing + related-projects See also -------- diff --git a/doc/related-projects.rst b/doc/related-projects.rst new file mode 100644 index 00000000000..9b75d0e1b3e --- /dev/null +++ b/doc/related-projects.rst @@ -0,0 +1,68 @@ +.. _related-projects: + +Xarray related projects +----------------------- + +Here below is a list of several existing libraries that build +functionality upon xarray. See also section :ref:`internals` for more +details on how to build xarray extensions. + +Geosciences +~~~~~~~~~~~ + +- `aospy `_: Automated analysis and management of gridded climate data. +- `infinite-diff `_: xarray-based finite-differencing, focused on gridded climate/meterology data +- `marc_analysis `_: Analysis package for CESM/MARC experiments and output. +- `MPAS-Analysis `_: Analysis for simulations produced with Model for Prediction Across Scales (MPAS) components and the Accelerated Climate Model for Energy (ACME). +- `OGGM `_: Open Global Glacier Model +- `Oocgcm `_: Analysis of large gridded geophysical datasets +- `Open Data Cube `_: Analysis toolkit of continental scale Earth Observation data from satellites. +- `Pangaea: `_: xarray extension for gridded land surface & weather model output). +- `Pangeo `_: A community effort for big data geoscience in the cloud. +- `PyGDX `_: Python 3 package for + accessing data stored in GAMS Data eXchange (GDX) files. Also uses a custom + subclass. +- `Regionmask `_: plotting and creation of masks of spatial regions +- `salem `_: Adds geolocalised subsetting, masking, and plotting operations to xarray's data structures via accessors. +- `Spyfit `_: FTIR spectroscopy of the atmosphere +- `windspharm `_: Spherical + harmonic wind analysis in Python. +- `wrf-python `_: A collection of diagnostic and interpolation routines for use with output of the Weather Research and Forecasting (WRF-ARW) Model. +- `xarray-simlab `_: xarray extension for computer model simulations. +- `xarray-topo `_: xarray extension for topographic analysis and modelling. +- `xbpch `_: xarray interface for bpch files. +- `xESMF `_: Universal Regridder for Geospatial Data. +- `xgcm `_: Extends the xarray data model to understand finite volume grid cells (common in General Circulation Models) and provides interpolation and difference operations for such grids. +- `xmitgcm `_: a python package for reading `MITgcm `_ binary MDS files into xarray data structures. +- `xshape `_: Tools for working with shapefiles, topographies, and polygons in xarray. +- `xskillscore `_: Metrics for verifying forecasts. + +Machine Learning +~~~~~~~~~~~~~~~~ +- `cesium `_: machine learning for time series analysis +- `Elm `_: Parallel machine learning on xarray data structures +- `sklearn-xarray (1) `_: Combines scikit-learn and xarray (1). +- `sklearn-xarray (2) `_: Combines scikit-learn and xarray (2). + +Extend xarray capabilities +~~~~~~~~~~~~~~~~~~~~~~~~~~ +- `Collocate `_: Collocate xarray trajectories in arbitrary physical dimensions +- `eofs `_: EOF analysis in Python. +- `xarray_extras `_: Advanced algorithms for xarray objects (e.g. intergrations/interpolations). +- `xrft `_: Fourier transforms for xarray data. +- `xr-scipy `_: A lightweight scipy wrapper for xarray. +- `X-regression `_: Multiple linear regression from Statsmodels library coupled with Xarray library. +- `xyzpy `_: Easily generate high dimensional data, including parallelization. + +Visualization +~~~~~~~~~~~~~ +- `Datashader `_, `geoviews `_, `holoviews `_, : visualization packages for large data. +- `hvplot `_ : A high-level plotting API for the PyData ecosystem built on HoloViews. +- `psyplot `_: Interactive data visualization with python. + +Other +~~~~~ +- `ptsa `_: EEG Time Series Analysis +- `pycalphad `_: Computational Thermodynamics in Python + +More projects can be found at the `"xarray" Github topic `_. From e3350fd724c30bb3695f755316f9b840445a0af6 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Tue, 14 Aug 2018 07:16:30 +0900 Subject: [PATCH 191/282] Raises a ValueError for a confliction between dimension names and level names (#2353) * Raises a ValueError for a confliction between dimension names and level names * Clean up whatsnew. --- doc/whats-new.rst | 8 +++++++- xarray/core/variable.py | 9 +++++++++ xarray/tests/test_dataset.py | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3641a072e2e..0c62cca6093 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -61,10 +61,16 @@ Bug fixes attribute being set. (:issue:`2201`) By `Thomas Voigt `_. + - Tests can be run in parallel with pytest-xdist -- Follow up the renamings in dask; from dask.ghost to dask.overlap + By `Tony Tung `_. + +- Now raises a ValueError when there is a conflict between dimension names and + level names of MultiIndex. (:issue:`2299`) By `Keisuke Fujii `_. +- Follow up the renamings in dask; from dask.ghost to dask.overlap + By `Keisuke Fujii `_. - Now :py:func:`xr.apply_ufunc` raises a ValueError when the size of ``input_core_dims`` is inconsistent with the number of arguments. diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 8b6a3d3bf21..d9772407b82 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1876,12 +1876,15 @@ def assert_unique_multiindex_level_names(variables): objects. """ level_names = defaultdict(list) + all_level_names = set() for var_name, var in variables.items(): if isinstance(var._data, PandasIndexAdapter): idx_level_names = var.to_index_variable().level_names if idx_level_names is not None: for n in idx_level_names: level_names[n].append('%r (%s)' % (n, var_name)) + if idx_level_names: + all_level_names.update(idx_level_names) for k, v in level_names.items(): if k in variables: @@ -1892,3 +1895,9 @@ def assert_unique_multiindex_level_names(variables): conflict_str = '\n'.join([', '.join(v) for v in duplicate_names]) raise ValueError('conflicting MultiIndex level name(s):\n%s' % conflict_str) + # Check confliction between level names and dimensions GH:2299 + for k, v in variables.items(): + for d in v.dims: + if d in all_level_names: + raise ValueError('conflicting level / dimension names. {} ' + 'already exists as a level name.'.format(d)) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index c0516ed7e56..08d71d462d8 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2456,6 +2456,18 @@ def test_assign_multiindex_level(self): with raises_regex(ValueError, 'conflicting MultiIndex'): data.assign(level_1=range(4)) data.assign_coords(level_1=range(4)) + # raise an Error when any level name is used as dimension GH:2299 + with pytest.raises(ValueError): + data['y'] = ('level_1', [0, 1]) + + def test_merge_multiindex_level(self): + data = create_test_multiindex() + other = Dataset({'z': ('level_1', [0, 1])}) # conflict dimension + with pytest.raises(ValueError): + data.merge(other) + other = Dataset({'level_1': ('x', [0, 1])}) # conflict variable name + with pytest.raises(ValueError): + data.merge(other) def test_setitem_original_non_unique_index(self): # regression test for GH943 From 4df048c146b8da7093faf96b3e59fb4d56945ec5 Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Mon, 13 Aug 2018 23:33:54 +0100 Subject: [PATCH 192/282] Remove redundant code from open_rasterio and ensure all transform tuples are six elements long (#2351) * Remove redundant chunk of code which sets the transform attribute when already set The transform attribute is already set further up in the open_rasterio function. This chunk of code just sets transform again - but actually sets it to a full nine-element tuple instead of the six-element tuple it should be (see comments around L245). * Added tests to check that the transform attribute read from rasterio Checks that this tuple is always six elements long --- xarray/backends/rasterio_.py | 7 ------- xarray/tests/test_backends.py | 5 +++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index e576435fcd2..5221cf0e913 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -259,13 +259,6 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, # Is the TIF tiled? (bool) # We cast it to an int for netCDF compatibility attrs['is_tiled'] = np.uint8(riods.value.is_tiled) - with warnings.catch_warnings(): - # casting riods.value.transform to a tuple makes this future proof - warnings.simplefilter('ignore', FutureWarning) - if hasattr(riods.value, 'transform'): - # Affine transformation matrix (tuple of floats) - # Describes coefficients mapping pixel coordinates to CRS - attrs['transform'] = tuple(riods.value.transform) if hasattr(riods.value, 'nodatavals'): # The nodata values for the raster bands attrs['nodatavals'] = tuple([np.nan if nodataval is None else nodataval diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index c90ca01cc11..a3d56b90954 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2809,6 +2809,7 @@ def test_utm(self): assert isinstance(rioda.attrs['res'], tuple) assert isinstance(rioda.attrs['is_tiled'], np.uint8) assert isinstance(rioda.attrs['transform'], tuple) + assert len(rioda.attrs['transform']) == 6 np.testing.assert_array_equal(rioda.attrs['nodatavals'], [np.NaN, np.NaN, np.NaN]) @@ -2830,6 +2831,7 @@ def test_non_rectilinear(self): assert isinstance(rioda.attrs['res'], tuple) assert isinstance(rioda.attrs['is_tiled'], np.uint8) assert isinstance(rioda.attrs['transform'], tuple) + assert len(rioda.attrs['transform']) == 6 # See if a warning is raised if we force it with self.assertWarns("transformation isn't rectilinear"): @@ -2849,6 +2851,7 @@ def test_platecarree(self): assert isinstance(rioda.attrs['res'], tuple) assert isinstance(rioda.attrs['is_tiled'], np.uint8) assert isinstance(rioda.attrs['transform'], tuple) + assert len(rioda.attrs['transform']) == 6 np.testing.assert_array_equal(rioda.attrs['nodatavals'], [-9765.]) @@ -2886,6 +2889,7 @@ def test_notransform(self): assert isinstance(rioda.attrs['res'], tuple) assert isinstance(rioda.attrs['is_tiled'], np.uint8) assert isinstance(rioda.attrs['transform'], tuple) + assert len(rioda.attrs['transform']) == 6 def test_indexing(self): with create_tmp_geotiff(8, 10, 3, transform_args=[1, 2, 0.5, 2.], @@ -3080,6 +3084,7 @@ def test_ENVI_tags(self): assert isinstance(rioda.attrs['res'], tuple) assert isinstance(rioda.attrs['is_tiled'], np.uint8) assert isinstance(rioda.attrs['transform'], tuple) + assert len(rioda.attrs['transform']) == 6 # from ENVI tags assert isinstance(rioda.attrs['description'], basestring) assert isinstance(rioda.attrs['map_info'], basestring) From df4a4b145b68c11d50576c961cf33c09b5bb9905 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 13 Aug 2018 21:46:33 -0600 Subject: [PATCH 193/282] Fix for zarr encoding bug (#2320) * simple fix for zarr encoding bug * remove test asserting error raised for bad chunk sizes * bump for ci --- doc/whats-new.rst | 4 ++++ xarray/backends/zarr.py | 5 ++--- xarray/tests/test_backends.py | 21 +++++++++++++++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0c62cca6093..bf9536a5fc7 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -61,6 +61,10 @@ Bug fixes attribute being set. (:issue:`2201`) By `Thomas Voigt `_. +- Fixed a bug in ``zarr`` backend which prevented use with datasets with + invalid chunk size encoding after reading from an existing store + (:issue:`2278`). + By `Joe Hamman `_. - Tests can be run in parallel with pytest-xdist By `Tony Tung `_. diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index a64ca858ed3..47b90c8a617 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -102,9 +102,8 @@ def _determine_zarr_chunks(enc_chunks, var_chunks, ndim): enc_chunks_tuple = tuple(enc_chunks) if len(enc_chunks_tuple) != ndim: - raise ValueError("zarr chunks tuple %r must have same length as " - "variable.ndim %g" % - (enc_chunks_tuple, ndim)) + # throw away encoding chunks, start over + return _determine_zarr_chunks(None, var_chunks, ndim) for x in enc_chunks_tuple: if not isinstance(x, int): diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a3d56b90954..e6de50b9dd2 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1404,12 +1404,6 @@ def test_chunk_encoding_with_dask(self): with self.roundtrip(ds_chunk4) as actual: self.assertEqual((4,), actual['var1'].encoding['chunks']) - # specify incompatible encoding - ds_chunk4['var1'].encoding.update({'chunks': (5, 5)}) - with pytest.raises(ValueError) as e_info: - with self.roundtrip(ds_chunk4) as actual: - pass - assert e_info.match('chunks') # TODO: remove this failure once syncronized overlapping writes are # supported by xarray @@ -1522,6 +1516,21 @@ def test_to_zarr_compute_false_roundtrip(self): with self.open(store) as actual: assert_identical(original, actual) + def test_encoding_chunksizes(self): + # regression test for GH2278 + # see also test_encoding_chunksizes_unlimited + nx, ny, nt = 4, 4, 5 + original = xr.Dataset({}, coords={'x': np.arange(nx), + 'y': np.arange(ny), + 't': np.arange(nt)}) + original['v'] = xr.Variable(('x', 'y', 't'), np.zeros((nx, ny, nt))) + original = original.chunk({'t': 1, 'x': 2, 'y': 2}) + + with self.roundtrip(original) as ds1: + assert_equal(ds1, original) + with self.roundtrip(ds1.isel(t=0)) as ds2: + assert_equal(ds2, original.isel(t=0)) + @requires_zarr class ZarrDictStoreTest(BaseZarrTest, TestCase): From b87b684b36cf5adbe4dca208aed0c69c44fc44c4 Mon Sep 17 00:00:00 2001 From: Brian Rose Date: Tue, 14 Aug 2018 14:44:21 -0400 Subject: [PATCH 194/282] Fix spelling -- change recieved to received (#2367) --- xarray/backends/api.py | 2 +- xarray/core/missing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index b2c0df7b01b..2bf13011bd1 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -806,7 +806,7 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, for obj in datasets: if not isinstance(obj, Dataset): raise TypeError('save_mfdataset only supports writing Dataset ' - 'objects, recieved type %s' % type(obj)) + 'objects, received type %s' % type(obj)) if groups is None: groups = [None] * len(datasets) diff --git a/xarray/core/missing.py b/xarray/core/missing.py index bec9e2e1931..232fa185c07 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -57,7 +57,7 @@ def __init__(self, xi, yi, method='linear', fill_value=None, **kwargs): if self.cons_kwargs: raise ValueError( - 'recieved invalid kwargs: %r' % self.cons_kwargs.keys()) + 'received invalid kwargs: %r' % self.cons_kwargs.keys()) if fill_value is None: self._left = np.nan From c27ca436321654a97e776aa0d055dfef357bc5a8 Mon Sep 17 00:00:00 2001 From: Maximilian Maahn Date: Tue, 14 Aug 2018 18:18:27 -0600 Subject: [PATCH 195/282] Faster unstack (#2364) * Make dataset.unstack faster by skipping reindex if not necessary. * Remove prints, add comment * added asv benchmark for unstacking * Added test * Simplified test * Added whats-new entry * PEP8 * Made asv test faster --- asv_bench/benchmarks/unstacking.py | 25 +++++++++++++++++++++++++ doc/whats-new.rst | 4 ++++ xarray/core/dataset.py | 9 +++++++-- xarray/tests/test_dataset.py | 15 ++++++++++++++- 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 asv_bench/benchmarks/unstacking.py diff --git a/asv_bench/benchmarks/unstacking.py b/asv_bench/benchmarks/unstacking.py new file mode 100644 index 00000000000..aa304d4eb40 --- /dev/null +++ b/asv_bench/benchmarks/unstacking.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import, division, print_function + +import numpy as np +import xarray as xr + +from . import requires_dask + + +class Unstacking(object): + def setup(self): + data = np.random.RandomState(0).randn(1, 1000, 500) + self.ds = xr.DataArray(data).stack(flat_dim=['dim_1', 'dim_2']) + + def time_unstack_fast(self): + self.ds.unstack('flat_dim') + + def time_unstack_slow(self): + self.ds[:, ::-1].unstack('flat_dim') + + +class UnstackingDask(Unstacking): + def setup(self, *args, **kwargs): + requires_dask() + super(UnstackingDask, self).setup(**kwargs) + self.ds = self.ds.chunk({'flat_dim': 50}) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index bf9536a5fc7..4552a4ca546 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -52,6 +52,10 @@ Enhancements (:issue:`2331`) By `Maximilian Roos `_. +- Applying ``unstack`` to a large DataArray or Dataset is now much faster if the MultiIndex has not been modified after stacking the indices. + (:issue:`1560`) + By `Maximilian Maahn `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 4b52178ad0e..e6bc2f8aeaf 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -2324,8 +2324,13 @@ def unstack(self, dim): 'a MultiIndex') full_idx = pd.MultiIndex.from_product(index.levels, names=index.names) - obj = self.reindex(copy=False, **{dim: full_idx}) - + + # take a shortcut in case the MultiIndex was not modified. + if index.equals(full_idx): + obj = self + else: + obj = self.reindex(copy=False, **{dim: full_idx}) + new_dim_names = index.names new_dim_sizes = [lev.size for lev in index.levels] diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 08d71d462d8..c67183db1ec 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2113,7 +2113,7 @@ def test_unstack_errors(self): with raises_regex(ValueError, 'does not have a MultiIndex'): ds.unstack('x') - def test_stack_unstack(self): + def test_stack_unstack_fast(self): ds = Dataset({'a': ('x', [0, 1]), 'b': (('x', 'y'), [[0, 1], [2, 3]]), 'x': [0, 1], @@ -2124,6 +2124,19 @@ def test_stack_unstack(self): actual = ds[['b']].stack(z=['x', 'y']).unstack('z') assert actual.identical(ds[['b']]) + def test_stack_unstack_slow(self): + ds = Dataset({'a': ('x', [0, 1]), + 'b': (('x', 'y'), [[0, 1], [2, 3]]), + 'x': [0, 1], + 'y': ['a', 'b']}) + stacked = ds.stack(z=['x', 'y']) + actual = stacked.isel(z=slice(None, None, -1)).unstack('z') + assert actual.broadcast_equals(ds) + + stacked = ds[['b']].stack(z=['x', 'y']) + actual = stacked.isel(z=slice(None, None, -1)).unstack('z') + assert actual.identical(ds[['b']]) + def test_update(self): data = create_test_data(seed=0) expected = data.copy() From cbb2aeb6492ad5364694396fb10e3b86abfe0aa6 Mon Sep 17 00:00:00 2001 From: Andrew Huang Date: Wed, 15 Aug 2018 01:11:28 -0700 Subject: [PATCH 196/282] Add option to not roll coords (#2360) * Add option to not roll coords * Rename keyword arg and add tests * Add what's new * Fix passing None and add more tests * Revise from comments * Revise with cleaner version * Revisions based on comments * Fix either_dict_or_kwargs * Revisions from comments --- doc/whats-new.rst | 5 ++++ xarray/core/dataarray.py | 15 ++++++++---- xarray/core/dataset.py | 44 ++++++++++++++++++++++++---------- xarray/tests/test_dataarray.py | 19 +++++++++++++-- xarray/tests/test_dataset.py | 30 ++++++++++++++++++++--- 5 files changed, 92 insertions(+), 21 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4552a4ca546..47d39b967e3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -56,6 +56,11 @@ Enhancements (:issue:`1560`) By `Maximilian Maahn `_. +- You can now control whether or not to offset the coordinates when using + the ``roll`` method and the current behavior, coordinates rolled by default, + raises a deprecation warning unless explicitly setting the keyword argument. + (:issue:`1875`) + By `Andrew Huang `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index f215bc47df8..b1be994416e 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -2015,14 +2015,20 @@ def shift(self, **shifts): variable = self.variable.shift(**shifts) return self._replace(variable) - def roll(self, **shifts): + def roll(self, shifts=None, roll_coords=None, **shifts_kwargs): """Roll this array by an offset along one or more dimensions. - Unlike shift, roll rotates all variables, including coordinates. The - direction of rotation is consistent with :py:func:`numpy.roll`. + Unlike shift, roll may rotate all variables, including coordinates + if specified. The direction of rotation is consistent with + :py:func:`numpy.roll`. Parameters ---------- + roll_coords : bool + Indicates whether to roll the coordinates by the offset + The current default of roll_coords (None, equivalent to True) is + deprecated and will change to False in a future version. + Explicitly pass roll_coords to silence the warning. **shifts : keyword arguments of the form {dim: offset} Integer offset to rotate each of the given dimensions. Positive offsets roll to the right; negative offsets roll to the left. @@ -2046,7 +2052,8 @@ def roll(self, **shifts): Coordinates: * x (x) int64 2 0 1 """ - ds = self._to_temp_dataset().roll(**shifts) + ds = self._to_temp_dataset().roll( + shifts=shifts, roll_coords=roll_coords, **shifts_kwargs) return self._from_temp_dataset(ds) @property diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index e6bc2f8aeaf..37544aca372 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -2324,13 +2324,13 @@ def unstack(self, dim): 'a MultiIndex') full_idx = pd.MultiIndex.from_product(index.levels, names=index.names) - + # take a shortcut in case the MultiIndex was not modified. if index.equals(full_idx): obj = self else: obj = self.reindex(copy=False, **{dim: full_idx}) - + new_dim_names = index.names new_dim_sizes = [lev.size for lev in index.levels] @@ -3360,18 +3360,28 @@ def shift(self, **shifts): return self._replace_vars_and_dims(variables) - def roll(self, **shifts): + def roll(self, shifts=None, roll_coords=None, **shifts_kwargs): """Roll this dataset by an offset along one or more dimensions. - Unlike shift, roll rotates all variables, including coordinates. The - direction of rotation is consistent with :py:func:`numpy.roll`. + Unlike shift, roll may rotate all variables, including coordinates + if specified. The direction of rotation is consistent with + :py:func:`numpy.roll`. Parameters ---------- - **shifts : keyword arguments of the form {dim: offset} - Integer offset to rotate each of the given dimensions. Positive - offsets roll to the right; negative offsets roll to the left. + shifts : dict, optional + A dict with keys matching dimensions and values given + by integers to rotate each of the given dimensions. Positive + offsets roll to the right; negative offsets roll to the left. + roll_coords : bool + Indicates whether to roll the coordinates by the offset + The current default of roll_coords (None, equivalent to True) is + deprecated and will change to False in a future version. + Explicitly pass roll_coords to silence the warning. + **shifts_kwargs : {dim: offset, ...}, optional + The keyword arguments form of ``shifts``. + One of shifts or shifts_kwargs must be provided. Returns ------- rolled : Dataset @@ -3394,15 +3404,25 @@ def roll(self, **shifts): Data variables: foo (x) object 'd' 'e' 'a' 'b' 'c' """ + shifts = either_dict_or_kwargs(shifts, shifts_kwargs, 'roll') invalid = [k for k in shifts if k not in self.dims] if invalid: raise ValueError("dimensions %r do not exist" % invalid) + if roll_coords is None: + warnings.warn("roll_coords will be set to False in the future." + " Explicitly set roll_coords to silence warning.", + FutureWarning, stacklevel=2) + roll_coords = True + + unrolled_vars = () if roll_coords else self.coords + variables = OrderedDict() - for name, var in iteritems(self.variables): - var_shifts = dict((k, v) for k, v in shifts.items() - if k in var.dims) - variables[name] = var.roll(**var_shifts) + for k, v in iteritems(self.variables): + if k not in unrolled_vars: + variables[k] = v.roll(**shifts) + else: + variables[k] = v return self._replace_vars_and_dims(variables) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 2950e97cc75..1a115192fb4 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3099,9 +3099,24 @@ def test_shift(self): actual = arr.shift(x=offset) assert_identical(expected, actual) - def test_roll(self): + def test_roll_coords(self): arr = DataArray([1, 2, 3], coords={'x': range(3)}, dims='x') - actual = arr.roll(x=1) + actual = arr.roll(x=1, roll_coords=True) + expected = DataArray([3, 1, 2], coords=[('x', [2, 0, 1])]) + assert_identical(expected, actual) + + def test_roll_no_coords(self): + arr = DataArray([1, 2, 3], coords={'x': range(3)}, dims='x') + actual = arr.roll(x=1, roll_coords=False) + expected = DataArray([3, 1, 2], coords=[('x', [0, 1, 2])]) + assert_identical(expected, actual) + + def test_roll_coords_none(self): + arr = DataArray([1, 2, 3], coords={'x': range(3)}, dims='x') + + with pytest.warns(FutureWarning): + actual = arr.roll(x=1, roll_coords=None) + expected = DataArray([3, 1, 2], coords=[('x', [2, 0, 1])]) assert_identical(expected, actual) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index c67183db1ec..e2a406b1e51 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -3862,18 +3862,42 @@ def test_shift(self): with raises_regex(ValueError, 'dimensions'): ds.shift(foo=123) - def test_roll(self): + def test_roll_coords(self): coords = {'bar': ('x', list('abc')), 'x': [-4, 3, 2]} attrs = {'meta': 'data'} ds = Dataset({'foo': ('x', [1, 2, 3])}, coords, attrs) - actual = ds.roll(x=1) + actual = ds.roll(x=1, roll_coords=True) ex_coords = {'bar': ('x', list('cab')), 'x': [2, -4, 3]} expected = Dataset({'foo': ('x', [3, 1, 2])}, ex_coords, attrs) assert_identical(expected, actual) with raises_regex(ValueError, 'dimensions'): - ds.roll(foo=123) + ds.roll(foo=123, roll_coords=True) + + def test_roll_no_coords(self): + coords = {'bar': ('x', list('abc')), 'x': [-4, 3, 2]} + attrs = {'meta': 'data'} + ds = Dataset({'foo': ('x', [1, 2, 3])}, coords, attrs) + actual = ds.roll(x=1, roll_coords=False) + + expected = Dataset({'foo': ('x', [3, 1, 2])}, coords, attrs) + assert_identical(expected, actual) + + with raises_regex(ValueError, 'dimensions'): + ds.roll(abc=321, roll_coords=False) + + def test_roll_coords_none(self): + coords = {'bar': ('x', list('abc')), 'x': [-4, 3, 2]} + attrs = {'meta': 'data'} + ds = Dataset({'foo': ('x', [1, 2, 3])}, coords, attrs) + + with pytest.warns(FutureWarning): + actual = ds.roll(x=1, roll_coords=None) + + ex_coords = {'bar': ('x', list('cab')), 'x': [2, -4, 3]} + expected = Dataset({'foo': ('x', [3, 1, 2])}, ex_coords, attrs) + assert_identical(expected, actual) def test_real_and_imag(self): attrs = {'foo': 'bar'} From 5155ef9ed2be7b3b201925c4902e8d633fec87a8 Mon Sep 17 00:00:00 2001 From: tv3141 Date: Thu, 16 Aug 2018 00:05:40 +0100 Subject: [PATCH 197/282] uncomment test (#2369) --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 951b151d829..0e51e946da0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,9 +95,7 @@ install: - python xarray/util/print_versions.py script: - # TODO: restore this check once the upstream pandas issue is fixed: - # https://github.com/pandas-dev/pandas/issues/21071 - # - python -OO -c "import xarray" + - python -OO -c "import xarray" - if [[ "$CONDA_ENV" == "docs" ]]; then conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc; sphinx-build -n -j auto -b html -d _build/doctrees doc _build/html; From 0b9ab2d12ae866a27050724d94facae6e56f5927 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Thu, 16 Aug 2018 15:59:32 +0900 Subject: [PATCH 198/282] Refactor nanops (#2236) * Inhouse nanops * Cleanup nanops * remove NAT_TYPES * flake8. * another flake8 * recover nat types * remove keep_dims option from nanops (to make them compatible with numpy==1.11). * Test aggregation over multiple dimensions * Remove print. * Docs. More cleanup. * flake8 * Bug fix. Better test coverage. * using isnull, where_method. Remove unnecessary conditional branching. * More refactoring based on the comments * remove dtype from nanmedian * Fix for nanmedian * Add tests for dataset * Add tests with resample. * lint * updated whatsnew * Revise from comments. * Use .any and .all method instead of np.any / np.all * Avoid using numpy methods * Avoid casting to int for dask array * Update whatsnew --- doc/whats-new.rst | 9 ++ xarray/core/common.py | 39 +++--- xarray/core/dtypes.py | 3 + xarray/core/duck_array_ops.py | 186 ++++++------------------- xarray/core/nanops.py | 208 ++++++++++++++++++++++++++++ xarray/core/nputils.py | 41 ++++++ xarray/core/ops.py | 14 +- xarray/tests/test_dataarray.py | 10 +- xarray/tests/test_dataset.py | 16 ++- xarray/tests/test_duck_array_ops.py | 172 +++++++++++++++++++++-- xarray/tests/test_variable.py | 4 +- 11 files changed, 519 insertions(+), 183 deletions(-) create mode 100644 xarray/core/nanops.py diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 47d39b967e3..4725fe74577 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,12 @@ Documentation Enhancements ~~~~~~~~~~~~ +- min_count option is newly supported in :py:meth:`~xarray.DataArray.sum`, + :py:meth:`~xarray.DataArray.prod` and :py:meth:`~xarray.Dataset.sum`, and + :py:meth:`~xarray.Dataset.prod`. + (:issue:`2230`) + By `Keisuke Fujii `_. + - :py:meth:`plot()` now accepts the kwargs ``xscale, yscale, xlim, ylim, xticks, yticks`` just like Pandas. Also ``xincrease=False, yincrease=False`` now use matplotlib's axis inverting methods instead of setting limits. By `Deepak Cherian `_. (:issue:`2224`) @@ -78,6 +84,9 @@ Bug fixes - Tests can be run in parallel with pytest-xdist By `Tony Tung `_. +- Follow up the renamings in dask; from dask.ghost to dask.overlap + By `Keisuke Fujii `_. + - Now raises a ValueError when there is a conflict between dimension names and level names of MultiIndex. (:issue:`2299`) By `Keisuke Fujii `_. diff --git a/xarray/core/common.py b/xarray/core/common.py index 3f934fcc769..55aca5f557f 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -2,6 +2,7 @@ import warnings from distutils.version import LooseVersion +from textwrap import dedent import numpy as np import pandas as pd @@ -27,20 +28,20 @@ def wrapped_func(self, dim=None, axis=None, keep_attrs=False, allow_lazy=True, **kwargs) return wrapped_func - _reduce_extra_args_docstring = \ - """dim : str or sequence of str, optional + _reduce_extra_args_docstring = dedent("""\ + dim : str or sequence of str, optional Dimension(s) over which to apply `{name}`. axis : int or sequence of int, optional Axis(es) over which to apply `{name}`. Only one of the 'dim' and 'axis' arguments can be supplied. If neither are supplied, then - `{name}` is calculated over axes.""" + `{name}` is calculated over axes.""") - _cum_extra_args_docstring = \ - """dim : str or sequence of str, optional + _cum_extra_args_docstring = dedent("""\ + dim : str or sequence of str, optional Dimension over which to apply `{name}`. axis : int or sequence of int, optional Axis over which to apply `{name}`. Only one of the 'dim' - and 'axis' arguments can be supplied.""" + and 'axis' arguments can be supplied.""") class ImplementsDatasetReduce(object): @@ -308,12 +309,12 @@ def assign_coords(self, **kwargs): assigned : same type as caller A new object with the new coordinates in addition to the existing data. - + Examples -------- - + Convert longitude coordinates from 0-359 to -180-179: - + >>> da = xr.DataArray(np.random.rand(4), ... coords=[np.array([358, 359, 0, 1])], ... dims='lon') @@ -445,11 +446,11 @@ def groupby(self, group, squeeze=True): grouped : GroupBy A `GroupBy` object patterned after `pandas.GroupBy` that can be iterated over in the form of `(unique_value, grouped_array)` pairs. - + Examples -------- Calculate daily anomalies for daily data: - + >>> da = xr.DataArray(np.linspace(0, 1826, num=1827), ... coords=[pd.date_range('1/1/2000', '31/12/2004', ... freq='D')], @@ -465,7 +466,7 @@ def groupby(self, group, squeeze=True): Coordinates: * time (time) datetime64[ns] 2000-01-01 2000-01-02 2000-01-03 ... dayofyear (time) int64 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 ... - + See Also -------- core.groupby.DataArrayGroupBy @@ -589,7 +590,7 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, closed=None, label=None, base=0, keep_attrs=False, **indexer): """Returns a Resample object for performing resampling operations. - Handles both downsampling and upsampling. If any intervals contain no + Handles both downsampling and upsampling. If any intervals contain no values from the original object, they will be given the value ``NaN``. Parameters @@ -616,11 +617,11 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, ------- resampled : same type as caller This object resampled. - + Examples -------- Downsample monthly time-series data to seasonal data: - + >>> da = xr.DataArray(np.linspace(0, 11, num=12), ... coords=[pd.date_range('15/12/1999', ... periods=12, freq=pd.DateOffset(months=1))], @@ -637,13 +638,13 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, * time (time) datetime64[ns] 1999-12-01 2000-03-01 2000-06-01 2000-09-01 Upsample monthly time-series data to daily data: - + >>> da.resample(time='1D').interpolate('linear') array([ 0. , 0.032258, 0.064516, ..., 10.935484, 10.967742, 11. ]) Coordinates: * time (time) datetime64[ns] 1999-12-15 1999-12-16 1999-12-17 ... - + References ---------- @@ -957,8 +958,8 @@ def contains_cftime_datetimes(var): sample = sample.item() return isinstance(sample, cftime_datetime) else: - return False - + return False + def _contains_datetime_like_objects(var): """Check if a variable contains datetime like objects (either diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 7326b936e2e..7ad44472f06 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -98,6 +98,9 @@ def maybe_promote(dtype): return np.dtype(dtype), fill_value +NAT_TYPES = (np.datetime64('NaT'), np.timedelta64('NaT')) + + def get_fill_value(dtype): """Return an appropriate fill value for this dtype. diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 3bd105064da..17eb310f8db 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -17,14 +17,6 @@ from .nputils import nanfirst, nanlast from .pycompat import dask_array_type -try: - import bottleneck as bn - has_bottleneck = True -except ImportError: - # use numpy methods instead - bn = np - has_bottleneck = False - try: import dask.array as dask_array from . import dask_array_compat @@ -175,7 +167,7 @@ def array_notnull_equiv(arr1, arr2): def count(data, axis=None): """Count the number of non-NA in this array along the given axis or axes """ - return sum(~isnull(data), axis=axis) + return np.sum(~isnull(data), axis=axis) def where(condition, x, y): @@ -213,159 +205,69 @@ def _ignore_warnings_if(condition): yield -def _nansum_object(value, axis=None, **kwargs): - """ In house nansum for object array """ - value = fillna(value, 0) - return _dask_or_eager_func('sum')(value, axis=axis, **kwargs) - - -def _nan_minmax_object(func, get_fill_value, value, axis=None, **kwargs): - """ In house nanmin and nanmax for object array """ - fill_value = get_fill_value(value.dtype) - valid_count = count(value, axis=axis) - filled_value = fillna(value, fill_value) - data = _dask_or_eager_func(func)(filled_value, axis=axis, **kwargs) - if not hasattr(data, 'dtype'): # scalar case - data = dtypes.fill_value(value.dtype) if valid_count == 0 else data - return np.array(data, dtype=value.dtype) - return where_method(data, valid_count != 0) - - -def _nan_argminmax_object(func, get_fill_value, value, axis=None, **kwargs): - """ In house nanargmin, nanargmax for object arrays. Always return integer - type """ - fill_value = get_fill_value(value.dtype) - valid_count = count(value, axis=axis) - value = fillna(value, fill_value) - data = _dask_or_eager_func(func)(value, axis=axis, **kwargs) - # dask seems return non-integer type - if isinstance(value, dask_array_type): - data = data.astype(int) - - if (valid_count == 0).any(): - raise ValueError('All-NaN slice encountered') - - return np.array(data, dtype=int) - - -def _nanmean_ddof_object(ddof, value, axis=None, **kwargs): - """ In house nanmean. ddof argument will be used in _nanvar method """ - valid_count = count(value, axis=axis) - value = fillna(value, 0) - # As dtype inference is impossible for object dtype, we assume float - # https://github.com/dask/dask/issues/3162 - dtype = kwargs.pop('dtype', None) - if dtype is None and value.dtype.kind == 'O': - dtype = value.dtype if value.dtype.kind in ['cf'] else float - - data = _dask_or_eager_func('sum')(value, axis=axis, dtype=dtype, **kwargs) - data = data / (valid_count - ddof) - return where_method(data, valid_count != 0) - - -def _nanvar_object(value, axis=None, **kwargs): - ddof = kwargs.pop('ddof', 0) - kwargs_mean = kwargs.copy() - kwargs_mean.pop('keepdims', None) - value_mean = _nanmean_ddof_object(ddof=0, value=value, axis=axis, - keepdims=True, **kwargs_mean) - squared = (value.astype(value_mean.dtype) - value_mean)**2 - return _nanmean_ddof_object(ddof, squared, axis=axis, **kwargs) - - -_nan_object_funcs = { - 'sum': _nansum_object, - 'min': partial(_nan_minmax_object, 'min', dtypes.get_pos_infinity), - 'max': partial(_nan_minmax_object, 'max', dtypes.get_neg_infinity), - 'argmin': partial(_nan_argminmax_object, 'argmin', - dtypes.get_pos_infinity), - 'argmax': partial(_nan_argminmax_object, 'argmax', - dtypes.get_neg_infinity), - 'mean': partial(_nanmean_ddof_object, 0), - 'var': _nanvar_object, -} - - -def _create_nan_agg_method(name, numeric_only=False, np_compat=False, - no_bottleneck=False, coerce_strings=False): +def _create_nan_agg_method(name, coerce_strings=False): + from . import nanops + def f(values, axis=None, skipna=None, **kwargs): if kwargs.pop('out', None) is not None: raise TypeError('`out` is not valid for {}'.format(name)) - # If dtype is supplied, we use numpy's method. - dtype = kwargs.get('dtype', None) values = asarray(values) - # dask requires dtype argument for object dtype - if (values.dtype == 'object' and name in ['sum', ]): - kwargs['dtype'] = values.dtype if dtype is None else dtype - if coerce_strings and values.dtype.kind in 'SU': values = values.astype(object) + func = None if skipna or (skipna is None and values.dtype.kind in 'cfO'): - if values.dtype.kind not in ['u', 'i', 'f', 'c']: - func = _nan_object_funcs.get(name, None) - using_numpy_nan_func = True - if func is None or values.dtype.kind not in 'Ob': - raise NotImplementedError( - 'skipna=True not yet implemented for %s with dtype %s' - % (name, values.dtype)) - else: - nanname = 'nan' + name - if (isinstance(axis, tuple) or not values.dtype.isnative or - no_bottleneck or (dtype is not None and - np.dtype(dtype) != values.dtype)): - # bottleneck can't handle multiple axis arguments or - # non-native endianness - if np_compat: - eager_module = npcompat - else: - eager_module = np - else: - kwargs.pop('dtype', None) - eager_module = bn - func = _dask_or_eager_func(nanname, eager_module) - using_numpy_nan_func = (eager_module is np or - eager_module is npcompat) + nanname = 'nan' + name + func = getattr(nanops, nanname) else: func = _dask_or_eager_func(name) - using_numpy_nan_func = False - with _ignore_warnings_if(using_numpy_nan_func): - try: - return func(values, axis=axis, **kwargs) - except AttributeError: - if isinstance(values, dask_array_type): - try: # dask/dask#3133 dask sometimes needs dtype argument - return func(values, axis=axis, dtype=values.dtype, - **kwargs) - except AttributeError: - msg = '%s is not yet implemented on dask arrays' % name - else: - assert using_numpy_nan_func - msg = ('%s is not available with skipna=False with the ' - 'installed version of numpy; upgrade to numpy 1.12 ' - 'or newer to use skipna=True or skipna=None' % name) - raise NotImplementedError(msg) - f.numeric_only = numeric_only + + try: + return func(values, axis=axis, **kwargs) + except AttributeError: + if isinstance(values, dask_array_type): + try: # dask/dask#3133 dask sometimes needs dtype argument + # if func does not accept dtype, then raises TypeError + return func(values, axis=axis, dtype=values.dtype, + **kwargs) + except (AttributeError, TypeError): + msg = '%s is not yet implemented on dask arrays' % name + else: + msg = ('%s is not available with skipna=False with the ' + 'installed version of numpy; upgrade to numpy 1.12 ' + 'or newer to use skipna=True or skipna=None' % name) + raise NotImplementedError(msg) + f.__name__ = name return f +# Attributes `numeric_only`, `available_min_count` is used for docs. +# See ops.inject_reduce_methods argmax = _create_nan_agg_method('argmax', coerce_strings=True) argmin = _create_nan_agg_method('argmin', coerce_strings=True) max = _create_nan_agg_method('max', coerce_strings=True) min = _create_nan_agg_method('min', coerce_strings=True) -sum = _create_nan_agg_method('sum', numeric_only=True) -mean = _create_nan_agg_method('mean', numeric_only=True) -std = _create_nan_agg_method('std', numeric_only=True) -var = _create_nan_agg_method('var', numeric_only=True) -median = _create_nan_agg_method('median', numeric_only=True) -prod = _create_nan_agg_method('prod', numeric_only=True, no_bottleneck=True) -cumprod_1d = _create_nan_agg_method( - 'cumprod', numeric_only=True, no_bottleneck=True) -cumsum_1d = _create_nan_agg_method( - 'cumsum', numeric_only=True, no_bottleneck=True) +sum = _create_nan_agg_method('sum') +sum.numeric_only = True +sum.available_min_count = True +mean = _create_nan_agg_method('mean') +mean.numeric_only = True +std = _create_nan_agg_method('std') +std.numeric_only = True +var = _create_nan_agg_method('var') +var.numeric_only = True +median = _create_nan_agg_method('median') +median.numeric_only = True +prod = _create_nan_agg_method('prod') +prod.numeric_only = True +sum.available_min_count = True +cumprod_1d = _create_nan_agg_method('cumprod') +cumprod_1d.numeric_only = True +cumsum_1d = _create_nan_agg_method('cumsum') +cumsum_1d.numeric_only = True def _nd_cum_func(cum_func, array, axis, **kwargs): diff --git a/xarray/core/nanops.py b/xarray/core/nanops.py new file mode 100644 index 00000000000..2309ed9619d --- /dev/null +++ b/xarray/core/nanops.py @@ -0,0 +1,208 @@ +from __future__ import absolute_import, division, print_function + +import numpy as np + +from . import dtypes +from .pycompat import dask_array_type +from . duck_array_ops import (count, isnull, fillna, where_method, + _dask_or_eager_func) +from . import nputils + +try: + import dask.array as dask_array +except ImportError: + dask_array = None + + +def _replace_nan(a, val): + """ + replace nan in a by val, and returns the replaced array and the nan + position + """ + mask = isnull(a) + return where_method(val, mask, a), mask + + +def _maybe_null_out(result, axis, mask, min_count=1): + """ + xarray version of pandas.core.nanops._maybe_null_out + """ + if hasattr(axis, '__len__'): # if tuple or list + raise ValueError('min_count is not available for reduction ' + 'with more than one dimensions.') + + if axis is not None and getattr(result, 'ndim', False): + null_mask = (mask.shape[axis] - mask.sum(axis) - min_count) < 0 + if null_mask.any(): + dtype, fill_value = dtypes.maybe_promote(result.dtype) + result = result.astype(dtype) + result[null_mask] = fill_value + + elif getattr(result, 'dtype', None) not in dtypes.NAT_TYPES: + null_mask = mask.size - mask.sum() + if null_mask < min_count: + result = np.nan + + return result + + +def _nan_argminmax_object(func, fill_value, value, axis=None, **kwargs): + """ In house nanargmin, nanargmax for object arrays. Always return integer + type + """ + valid_count = count(value, axis=axis) + value = fillna(value, fill_value) + data = _dask_or_eager_func(func)(value, axis=axis, **kwargs) + + # TODO This will evaluate dask arrays and might be costly. + if (valid_count == 0).any(): + raise ValueError('All-NaN slice encountered') + + return data + + +def _nan_minmax_object(func, fill_value, value, axis=None, **kwargs): + """ In house nanmin and nanmax for object array """ + valid_count = count(value, axis=axis) + filled_value = fillna(value, fill_value) + data = getattr(np, func)(filled_value, axis=axis, **kwargs) + if not hasattr(data, 'dtype'): # scalar case + data = dtypes.fill_value(value.dtype) if valid_count == 0 else data + return np.array(data, dtype=value.dtype) + return where_method(data, valid_count != 0) + + +def nanmin(a, axis=None, out=None): + if a.dtype.kind == 'O': + return _nan_minmax_object( + 'min', dtypes.get_pos_infinity(a.dtype), a, axis) + + module = dask_array if isinstance(a, dask_array_type) else nputils + return module.nanmin(a, axis=axis) + + +def nanmax(a, axis=None, out=None): + if a.dtype.kind == 'O': + return _nan_minmax_object( + 'max', dtypes.get_neg_infinity(a.dtype), a, axis) + + module = dask_array if isinstance(a, dask_array_type) else nputils + return module.nanmax(a, axis=axis) + + +def nanargmin(a, axis=None): + fill_value = dtypes.get_pos_infinity(a.dtype) + if a.dtype.kind == 'O': + return _nan_argminmax_object('argmin', fill_value, a, axis=axis) + a, mask = _replace_nan(a, fill_value) + if isinstance(a, dask_array_type): + res = dask_array.argmin(a, axis=axis) + else: + res = np.argmin(a, axis=axis) + + if mask is not None: + mask = mask.all(axis=axis) + if mask.any(): + raise ValueError("All-NaN slice encountered") + return res + + +def nanargmax(a, axis=None): + fill_value = dtypes.get_neg_infinity(a.dtype) + if a.dtype.kind == 'O': + return _nan_argminmax_object('argmax', fill_value, a, axis=axis) + + a, mask = _replace_nan(a, fill_value) + if isinstance(a, dask_array_type): + res = dask_array.argmax(a, axis=axis) + else: + res = np.argmax(a, axis=axis) + + if mask is not None: + mask = mask.all(axis=axis) + if mask.any(): + raise ValueError("All-NaN slice encountered") + return res + + +def nansum(a, axis=None, dtype=None, out=None, min_count=None): + a, mask = _replace_nan(a, 0) + result = _dask_or_eager_func('sum')(a, axis=axis, dtype=dtype) + if min_count is not None: + return _maybe_null_out(result, axis, mask, min_count) + else: + return result + + +def _nanmean_ddof_object(ddof, value, axis=None, **kwargs): + """ In house nanmean. ddof argument will be used in _nanvar method """ + from .duck_array_ops import (count, fillna, _dask_or_eager_func, + where_method) + + valid_count = count(value, axis=axis) + value = fillna(value, 0) + # As dtype inference is impossible for object dtype, we assume float + # https://github.com/dask/dask/issues/3162 + dtype = kwargs.pop('dtype', None) + if dtype is None and value.dtype.kind == 'O': + dtype = value.dtype if value.dtype.kind in ['cf'] else float + + data = _dask_or_eager_func('sum')(value, axis=axis, dtype=dtype, **kwargs) + data = data / (valid_count - ddof) + return where_method(data, valid_count != 0) + + +def nanmean(a, axis=None, dtype=None, out=None): + if a.dtype.kind == 'O': + return _nanmean_ddof_object(0, a, axis=axis, dtype=dtype) + + if isinstance(a, dask_array_type): + return dask_array.nanmean(a, axis=axis, dtype=dtype) + + return np.nanmean(a, axis=axis, dtype=dtype) + + +def nanmedian(a, axis=None, out=None): + return _dask_or_eager_func('nanmedian', eager_module=nputils)(a, axis=axis) + + +def _nanvar_object(value, axis=None, **kwargs): + ddof = kwargs.pop('ddof', 0) + kwargs_mean = kwargs.copy() + kwargs_mean.pop('keepdims', None) + value_mean = _nanmean_ddof_object(ddof=0, value=value, axis=axis, + keepdims=True, **kwargs_mean) + squared = (value.astype(value_mean.dtype) - value_mean)**2 + return _nanmean_ddof_object(ddof, squared, axis=axis, **kwargs) + + +def nanvar(a, axis=None, dtype=None, out=None, ddof=0): + if a.dtype.kind == 'O': + return _nanvar_object(a, axis=axis, dtype=dtype, ddof=ddof) + + return _dask_or_eager_func('nanvar', eager_module=nputils)( + a, axis=axis, dtype=dtype, ddof=ddof) + + +def nanstd(a, axis=None, dtype=None, out=None): + return _dask_or_eager_func('nanstd', eager_module=nputils)( + a, axis=axis, dtype=dtype) + + +def nanprod(a, axis=None, dtype=None, out=None, min_count=None): + a, mask = _replace_nan(a, 1) + result = _dask_or_eager_func('nanprod')(a, axis=axis, dtype=dtype, out=out) + if min_count is not None: + return _maybe_null_out(result, axis, mask, min_count) + else: + return result + + +def nancumsum(a, axis=None, dtype=None, out=None): + return _dask_or_eager_func('nancumsum', eager_module=nputils)( + a, axis=axis, dtype=dtype) + + +def nancumprod(a, axis=None, dtype=None, out=None): + return _dask_or_eager_func('nancumprod', eager_module=nputils)( + a, axis=axis, dtype=dtype) diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 6df2d34bfe3..a8d596abd86 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -5,6 +5,14 @@ import numpy as np import pandas as pd +try: + import bottleneck as bn + _USE_BOTTLENECK = True +except ImportError: + # use numpy methods instead + bn = np + _USE_BOTTLENECK = False + def _validate_axis(data, axis): ndim = data.ndim @@ -195,3 +203,36 @@ def _rolling_window(a, window, axis=-1): rolling = np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides, writeable=False) return np.swapaxes(rolling, -2, axis) + + +def _create_bottleneck_method(name, npmodule=np): + def f(values, axis=None, **kwds): + dtype = kwds.get('dtype', None) + bn_func = getattr(bn, name, None) + + if (_USE_BOTTLENECK and bn_func is not None and + not isinstance(axis, tuple) and + values.dtype.kind in 'uifc' and + values.dtype.isnative and + (dtype is None or np.dtype(dtype) == values.dtype)): + # bottleneck does not take care dtype, min_count + kwds.pop('dtype', None) + result = bn_func(values, axis=axis, **kwds) + else: + result = getattr(npmodule, name)(values, axis=axis, **kwds) + + return result + + f.__name__ = name + return f + + +nanmin = _create_bottleneck_method('nanmin') +nanmax = _create_bottleneck_method('nanmax') +nanmean = _create_bottleneck_method('nanmean') +nanmedian = _create_bottleneck_method('nanmedian') +nanvar = _create_bottleneck_method('nanvar') +nanstd = _create_bottleneck_method('nanstd') +nanprod = _create_bottleneck_method('nanprod') +nancumsum = _create_bottleneck_method('nancumsum') +nancumprod = _create_bottleneck_method('nancumprod') diff --git a/xarray/core/ops.py b/xarray/core/ops.py index d9e8ceb65d5..a0dd2212a8f 100644 --- a/xarray/core/ops.py +++ b/xarray/core/ops.py @@ -86,7 +86,7 @@ If True, skip missing values (as marked by NaN). By default, only skips missing values for float dtypes; other dtypes either do not have a sentinel missing value (int) or skipna=True has not been - implemented (object, datetime64 or timedelta64). + implemented (object, datetime64 or timedelta64).{min_count_docs} keep_attrs : bool, optional If True, the attributes (`attrs`) will be copied from the original object to the new one. If False (default), the new object will be @@ -102,6 +102,12 @@ indicated dimension(s) removed. """ +_MINCOUNT_DOCSTRING = """ +min_count : int, default None + The required number of valid values to perform the operation. + If fewer than min_count non-NA values are present the result will + be NA. New in version 0.10.8: Added with the default being None.""" + _ROLLING_REDUCE_DOCSTRING_TEMPLATE = """\ Reduce this {da_or_ds}'s data windows by applying `{name}` along its dimension. @@ -236,11 +242,15 @@ def inject_reduce_methods(cls): [('count', duck_array_ops.count, False)]) for name, f, include_skipna in methods: numeric_only = getattr(f, 'numeric_only', False) + available_min_count = getattr(f, 'available_min_count', False) + min_count_docs = _MINCOUNT_DOCSTRING if available_min_count else '' + func = cls._reduce_method(f, include_skipna, numeric_only) func.__name__ = name func.__doc__ = _REDUCE_DOCSTRING_TEMPLATE.format( name=name, cls=cls.__name__, - extra_args=cls._reduce_extra_args_docstring.format(name=name)) + extra_args=cls._reduce_extra_args_docstring.format(name=name), + min_count_docs=min_count_docs) setattr(cls, name, func) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 1a115192fb4..3619688d091 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3344,7 +3344,9 @@ def test_isin(da): def test_rolling_iter(da): rolling_obj = da.rolling(time=7) - rolling_obj_mean = rolling_obj.mean() + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Mean of empty slice') + rolling_obj_mean = rolling_obj.mean() assert len(rolling_obj.window_labels) == len(da['time']) assert_identical(rolling_obj.window_labels, da['time']) @@ -3352,8 +3354,10 @@ def test_rolling_iter(da): for i, (label, window_da) in enumerate(rolling_obj): assert label == da['time'].isel(time=i) - actual = rolling_obj_mean.isel(time=i) - expected = window_da.mean('time') + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Mean of empty slice') + actual = rolling_obj_mean.isel(time=i) + expected = window_da.mean('time') # TODO add assert_allclose_with_nan, which compares nan position # as well as the closeness of the values. diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index e2a406b1e51..d73632c10a7 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2740,6 +2740,20 @@ def test_resample_and_first(self): result = actual.reduce(method) assert_equal(expected, result) + def test_resample_min_count(self): + times = pd.date_range('2000-01-01', freq='6H', periods=10) + ds = Dataset({'foo': (['time', 'x', 'y'], np.random.randn(10, 5, 3)), + 'bar': ('time', np.random.randn(10), {'meta': 'data'}), + 'time': times}) + # inject nan + ds['foo'] = xr.where(ds['foo'] > 2.0, np.nan, ds['foo']) + + actual = ds.resample(time='1D').sum(min_count=1) + expected = xr.concat([ + ds.isel(time=slice(i * 4, (i + 1) * 4)).sum('time', min_count=1) + for i in range(3)], dim=actual['time']) + assert_equal(expected, actual) + def test_resample_by_mean_with_keep_attrs(self): times = pd.date_range('2000-01-01', freq='6H', periods=10) ds = Dataset({'foo': (['time', 'x', 'y'], np.random.randn(10, 5, 3)), @@ -3378,7 +3392,6 @@ def test_reduce(self): (('dim2', 'time'), ['dim1', 'dim3']), ((), ['dim1', 'dim2', 'dim3', 'time'])]: actual = data.min(dim=reduct).dims - print(reduct, actual, expected) self.assertItemsEqual(actual, expected) assert_equal(data.mean(dim=[]), data) @@ -3433,7 +3446,6 @@ def test_reduce_cumsum_test_dims(self): ('time', ['dim1', 'dim2', 'dim3']) ]: actual = getattr(data, cumfunc)(dim=reduct).dims - print(reduct, actual, expected) self.assertItemsEqual(actual, expected) def test_reduce_non_numeric(self): diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index f3f93491822..3f32fc49fd2 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -1,12 +1,16 @@ from __future__ import absolute_import, division, print_function +from distutils.version import LooseVersion + import numpy as np +import pandas as pd import pytest +from textwrap import dedent from numpy import array, nan import warnings -from xarray import DataArray, concat -from xarray.core import duck_array_ops +from xarray import DataArray, Dataset, concat +from xarray.core import duck_array_ops, dtypes from xarray.core.duck_array_ops import ( array_notnull_equiv, concatenate, count, first, last, mean, rolling_window, stack, where) @@ -100,7 +104,10 @@ def test_concatenate_type_promotion(self): assert_array_equal(result, np.array([1, 'b'], dtype=object)) def test_all_nan_arrays(self): - assert np.isnan(mean([np.nan, np.nan])) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'All-NaN slice') + warnings.filterwarnings('ignore', 'Mean of empty slice') + assert np.isnan(mean([np.nan, np.nan])) def test_cumsum_1d(): @@ -197,10 +204,15 @@ def construct_dataarray(dim_num, dtype, contains_nan, dask): array = rng.choice(['a', 'b', 'c', 'd'], size=shapes) else: raise ValueError - da = DataArray(array, dims=dims, coords={'x': np.arange(16)}, name='da') if contains_nan: - da = da.reindex(x=np.arange(20)) + inds = rng.choice(range(array.size), int(array.size * 0.2)) + dtype, fill_value = dtypes.maybe_promote(array.dtype) + array = array.astype(dtype) + array.flat[inds] = fill_value + + da = DataArray(array, dims=dims, coords={'x': np.arange(16)}, name='da') + if dask and has_dask: chunks = {d: 4 for d in dims} da = da.chunk(chunks) @@ -234,10 +246,16 @@ def series_reduce(da, func, dim, **kwargs): return concat(da1, dim=d) +def assert_dask_array(da, dask): + if dask and da.ndim > 0: + assert isinstance(da.data, dask_array_type) + + @pytest.mark.parametrize('dim_num', [1, 2]) @pytest.mark.parametrize('dtype', [float, int, np.float32, np.bool_]) @pytest.mark.parametrize('dask', [False, True]) @pytest.mark.parametrize('func', ['sum', 'min', 'max', 'mean', 'var']) +# TODO test cumsum, cumprod @pytest.mark.parametrize('skipna', [False, True]) @pytest.mark.parametrize('aggdim', [None, 'x']) def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): @@ -251,6 +269,9 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): if dask and not has_dask: pytest.skip('requires dask') + if dask and skipna is False and dtype in [np.bool_]: + pytest.skip('dask does not compute object-typed array') + rtol = 1e-04 if dtype == np.float32 else 1e-05 da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask) @@ -259,6 +280,7 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): # TODO: remove these after resolving # https://github.com/dask/dask/issues/3245 with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Mean of empty slice') warnings.filterwarnings('ignore', 'All-NaN slice') warnings.filterwarnings('ignore', 'invalid value encountered in') @@ -272,6 +294,7 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): expected = getattr(np, func)(da.values, axis=axis) actual = getattr(da, func)(skipna=skipna, dim=aggdim) + assert_dask_array(actual, dask) assert np.allclose(actual.values, np.array(expected), rtol=1.0e-4, equal_nan=True) except (TypeError, AttributeError, ZeroDivisionError): @@ -279,14 +302,21 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): # nanmean for object dtype pass - # make sure the compatiblility with pandas' results. actual = getattr(da, func)(skipna=skipna, dim=aggdim) + + # for dask case, make sure the result is the same for numpy backend + expected = getattr(da.compute(), func)(skipna=skipna, dim=aggdim) + assert_allclose(actual, expected, rtol=rtol) + + # make sure the compatiblility with pandas' results. if func == 'var': expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=0) assert_allclose(actual, expected, rtol=rtol) # also check ddof!=0 case actual = getattr(da, func)(skipna=skipna, dim=aggdim, ddof=5) + if dask: + assert isinstance(da.data, dask_array_type) expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=5) assert_allclose(actual, expected, rtol=rtol) @@ -297,11 +327,14 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): # make sure the dtype argument if func not in ['max', 'min']: actual = getattr(da, func)(skipna=skipna, dim=aggdim, dtype=float) + assert_dask_array(actual, dask) assert actual.dtype == float # without nan da = construct_dataarray(dim_num, dtype, contains_nan=False, dask=dask) actual = getattr(da, func)(skipna=skipna) + if dask: + assert isinstance(da.data, dask_array_type) expected = getattr(np, 'nan{}'.format(func))(da.values) if actual.dtype == object: assert actual.values == np.array(expected) @@ -338,13 +371,6 @@ def test_argmin_max(dim_num, dtype, contains_nan, dask, func, skipna, aggdim): with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'All-NaN slice') - if aggdim == 'y' and contains_nan and skipna: - with pytest.raises(ValueError): - actual = da.isel(**{ - aggdim: getattr(da, 'arg' + func)( - dim=aggdim, skipna=skipna).compute()}) - return - actual = da.isel(**{aggdim: getattr(da, 'arg' + func) (dim=aggdim, skipna=skipna).compute()}) expected = getattr(da, func)(dim=aggdim, skipna=skipna) @@ -354,6 +380,7 @@ def test_argmin_max(dim_num, dtype, contains_nan, dask, func, skipna, aggdim): def test_argmin_max_error(): da = construct_dataarray(2, np.bool_, contains_nan=True, dask=False) + da[0] = np.nan with pytest.raises(ValueError): da.argmin(dim='y') @@ -388,3 +415,122 @@ def test_dask_rolling(axis, window, center): with pytest.raises(ValueError): rolling_window(dx, axis=axis, window=100, center=center, fill_value=np.nan) + + +@pytest.mark.parametrize('dim_num', [1, 2]) +@pytest.mark.parametrize('dtype', [float, int, np.float32, np.bool_]) +@pytest.mark.parametrize('dask', [False, True]) +@pytest.mark.parametrize('func', ['sum', 'prod']) +@pytest.mark.parametrize('aggdim', [None, 'x']) +def test_min_count(dim_num, dtype, dask, func, aggdim): + if dask and not has_dask: + pytest.skip('requires dask') + + da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask) + min_count = 3 + + actual = getattr(da, func)(dim=aggdim, skipna=True, min_count=min_count) + + if LooseVersion(pd.__version__) >= LooseVersion('0.22.0'): + # min_count is only implenented in pandas > 0.22 + expected = series_reduce(da, func, skipna=True, dim=aggdim, + min_count=min_count) + assert_allclose(actual, expected) + + assert_dask_array(actual, dask) + + +@pytest.mark.parametrize('func', ['sum', 'prod']) +def test_min_count_dataset(func): + da = construct_dataarray(2, dtype=float, contains_nan=True, dask=False) + ds = Dataset({'var1': da}, coords={'scalar': 0}) + actual = getattr(ds, func)(dim='x', skipna=True, min_count=3)['var1'] + expected = getattr(ds['var1'], func)(dim='x', skipna=True, min_count=3) + assert_allclose(actual, expected) + + +@pytest.mark.parametrize('dtype', [float, int, np.float32, np.bool_]) +@pytest.mark.parametrize('dask', [False, True]) +@pytest.mark.parametrize('func', ['sum', 'prod']) +def test_multiple_dims(dtype, dask, func): + if dask and not has_dask: + pytest.skip('requires dask') + da = construct_dataarray(3, dtype, contains_nan=True, dask=dask) + + actual = getattr(da, func)(('x', 'y')) + expected = getattr(getattr(da, func)('x'), func)('y') + assert_allclose(actual, expected) + + +def test_docs(): + # with min_count + actual = DataArray.sum.__doc__ + expected = dedent("""\ + Reduce this DataArray's data by applying `sum` along some dimension(s). + + Parameters + ---------- + dim : str or sequence of str, optional + Dimension(s) over which to apply `sum`. + axis : int or sequence of int, optional + Axis(es) over which to apply `sum`. Only one of the 'dim' + and 'axis' arguments can be supplied. If neither are supplied, then + `sum` is calculated over axes. + skipna : bool, optional + If True, skip missing values (as marked by NaN). By default, only + skips missing values for float dtypes; other dtypes either do not + have a sentinel missing value (int) or skipna=True has not been + implemented (object, datetime64 or timedelta64). + min_count : int, default None + The required number of valid values to perform the operation. + If fewer than min_count non-NA values are present the result will + be NA. New in version 0.10.8: Added with the default being None. + keep_attrs : bool, optional + If True, the attributes (`attrs`) will be copied from the original + object to the new one. If False (default), the new object will be + returned without attributes. + **kwargs : dict + Additional keyword arguments passed on to the appropriate array + function for calculating `sum` on this object's data. + + Returns + ------- + reduced : DataArray + New DataArray object with `sum` applied to its data and the + indicated dimension(s) removed. + """) + assert actual == expected + + # without min_count + actual = DataArray.std.__doc__ + expected = dedent("""\ + Reduce this DataArray's data by applying `std` along some dimension(s). + + Parameters + ---------- + dim : str or sequence of str, optional + Dimension(s) over which to apply `std`. + axis : int or sequence of int, optional + Axis(es) over which to apply `std`. Only one of the 'dim' + and 'axis' arguments can be supplied. If neither are supplied, then + `std` is calculated over axes. + skipna : bool, optional + If True, skip missing values (as marked by NaN). By default, only + skips missing values for float dtypes; other dtypes either do not + have a sentinel missing value (int) or skipna=True has not been + implemented (object, datetime64 or timedelta64). + keep_attrs : bool, optional + If True, the attributes (`attrs`) will be copied from the original + object to the new one. If False (default), the new object will be + returned without attributes. + **kwargs : dict + Additional keyword arguments passed on to the appropriate array + function for calculating `std` on this object's data. + + Returns + ------- + reduced : DataArray + New DataArray object with `std` applied to its data and the + indicated dimension(s) removed. + """) + assert actual == expected diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index cdb578aff6c..3db5e6adc4b 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1503,8 +1503,8 @@ def test_reduce_funcs(self): assert_identical(v.all(dim='x'), Variable([], False)) v = Variable('t', pd.date_range('2000-01-01', periods=3)) - with pytest.raises(NotImplementedError): - v.argmax(skipna=True) + assert v.argmax(skipna=True) == 2 + assert_identical( v.max(), Variable([], pd.Timestamp('2000-01-03'))) From 725bd57ffa64d7e391ceef2b056fa8122ec09e8d Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Mon, 20 Aug 2018 10:12:36 +0900 Subject: [PATCH 199/282] More support of non-string dimension names (#2373) * More support for non-string dimension. * Avoid using kwargs in rolling. * Restore assign_coords, fixes typo --- xarray/core/common.py | 24 +++++++------ xarray/core/dataarray.py | 55 ++++++++++++++++++---------- xarray/core/dataset.py | 78 ++++++++++++++++++++++++++-------------- xarray/core/rolling.py | 53 ++++++++++++++------------- xarray/core/variable.py | 47 ++++++++++++++++++------ 5 files changed, 162 insertions(+), 95 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index 55aca5f557f..280034a30dd 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -10,7 +10,7 @@ from . import duck_array_ops, dtypes, formatting, ops from .arithmetic import SupportsArithmetic from .pycompat import OrderedDict, basestring, dask_array_type, suppress -from .utils import Frozen, SortedKeysDict +from .utils import either_dict_or_kwargs, Frozen, SortedKeysDict class ImplementsArrayReduce(object): @@ -526,24 +526,24 @@ def groupby_bins(self, group, bins, right=True, labels=None, precision=3, 'precision': precision, 'include_lowest': include_lowest}) - def rolling(self, min_periods=None, center=False, **windows): + def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): """ Rolling window object. Parameters ---------- + dim: dict, optional + Mapping from the dimension name to create the rolling iterator + along (e.g. `time`) to its moving window size. min_periods : int, default None Minimum number of observations in window required to have a value (otherwise result is NA). The default, None, is equivalent to setting min_periods equal to the size of the window. center : boolean, default False Set the labels at the center of the window. - **windows : dim=window - dim : str - Name of the dimension to create the rolling iterator - along (e.g., `time`). - window : int - Size of the moving window. + **dim_kwargs : optional + The keyword arguments form of ``dim``. + One of dim or dim_kwarg must be provided. Returns ------- @@ -582,9 +582,9 @@ def rolling(self, min_periods=None, center=False, **windows): core.rolling.DataArrayRolling core.rolling.DatasetRolling """ - - return self._rolling_cls(self, min_periods=min_periods, - center=center, **windows) + dim = either_dict_or_kwargs(dim, dim_kwargs, 'rolling') + return self._rolling_cls(self, dim, min_periods=min_periods, + center=center) def resample(self, freq=None, dim=None, how=None, skipna=None, closed=None, label=None, base=0, keep_attrs=False, **indexer): @@ -650,6 +650,8 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, .. [1] http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases """ + # TODO support non-string indexer after removing the old API. + from .dataarray import DataArray from .resample import RESAMPLE_DIM diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index b1be994416e..359812f2cc3 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -779,9 +779,9 @@ def sel(self, indexers=None, method=None, tolerance=None, drop=False, DataArray.isel """ - indexers = either_dict_or_kwargs(indexers, indexers_kwargs, 'sel') ds = self._to_temp_dataset().sel( - indexers=indexers, drop=drop, method=method, tolerance=tolerance) + indexers=indexers, drop=drop, method=method, tolerance=tolerance, + **indexers_kwargs) return self._from_temp_dataset(ds) def isel_points(self, dim='points', **indexers): @@ -1092,22 +1092,26 @@ def expand_dims(self, dim, axis=None): ds = self._to_temp_dataset().expand_dims(dim, axis) return self._from_temp_dataset(ds) - def set_index(self, append=False, inplace=False, **indexes): + def set_index(self, indexes=None, append=False, inplace=False, + **indexes_kwargs): """Set DataArray (multi-)indexes using one or more existing coordinates. Parameters ---------- + indexes : {dim: index, ...} + Mapping from names matching dimensions and values given + by (lists of) the names of existing coordinates or variables to set + as new (multi-)index. append : bool, optional If True, append the supplied index(es) to the existing index(es). Otherwise replace the existing index(es) (default). inplace : bool, optional If True, set new index(es) in-place. Otherwise, return a new DataArray object. - **indexes : {dim: index, ...} - Keyword arguments with names matching dimensions and values given - by (lists of) the names of existing coordinates or variables to set - as new (multi-)index. + **indexes_kwargs: optional + The keyword arguments form of ``indexes``. + One of indexes or indexes_kwargs must be provided. Returns ------- @@ -1118,6 +1122,7 @@ def set_index(self, append=False, inplace=False, **indexes): -------- DataArray.reset_index """ + indexes = either_dict_or_kwargs(indexes, indexes_kwargs, 'set_index') coords, _ = merge_indexes(indexes, self._coords, set(), append=append) if inplace: self._coords = coords @@ -1156,18 +1161,22 @@ def reset_index(self, dims_or_levels, drop=False, inplace=False): else: return self._replace(coords=coords) - def reorder_levels(self, inplace=False, **dim_order): + def reorder_levels(self, dim_order=None, inplace=False, + **dim_order_kwargs): """Rearrange index levels using input order. Parameters ---------- + dim_order : optional + Mapping from names matching dimensions and values given + by lists representing new level orders. Every given dimension + must have a multi-index. inplace : bool, optional If True, modify the dataarray in-place. Otherwise, return a new DataArray object. - **dim_order : optional - Keyword arguments with names matching dimensions and values given - by lists representing new level orders. Every given dimension - must have a multi-index. + **dim_order_kwargs: optional + The keyword arguments form of ``dim_order``. + One of dim_order or dim_order_kwargs must be provided. Returns ------- @@ -1175,6 +1184,8 @@ def reorder_levels(self, inplace=False, **dim_order): Another dataarray, with this dataarray's data but replaced coordinates. """ + dim_order = either_dict_or_kwargs(dim_order, dim_order_kwargs, + 'reorder_levels') replace_coords = {} for dim, order in dim_order.items(): coord = self._coords[dim] @@ -1190,7 +1201,7 @@ def reorder_levels(self, inplace=False, **dim_order): else: return self._replace(coords=coords) - def stack(self, **dimensions): + def stack(self, dimensions=None, **dimensions_kwargs): """ Stack any number of existing dimensions into a single new dimension. @@ -1199,9 +1210,12 @@ def stack(self, **dimensions): Parameters ---------- - **dimensions : keyword arguments of the form new_name=(dim1, dim2, ...) + dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they replace. + **dimensions_kwargs: + The keyword arguments form of ``dimensions``. + One of dimensions or dimensions_kwargs must be provided. Returns ------- @@ -1230,7 +1244,7 @@ def stack(self, **dimensions): -------- DataArray.unstack """ - ds = self._to_temp_dataset().stack(**dimensions) + ds = self._to_temp_dataset().stack(dimensions, **dimensions_kwargs) return self._from_temp_dataset(ds) def unstack(self, dim): @@ -1978,7 +1992,7 @@ def diff(self, dim, n=1, label='upper'): ds = self._to_temp_dataset().diff(n=n, dim=dim, label=label) return self._from_temp_dataset(ds) - def shift(self, **shifts): + def shift(self, shifts=None, **shifts_kwargs): """Shift this array by an offset along one or more dimensions. Only the data is moved; coordinates stay in place. Values shifted from @@ -1987,10 +2001,13 @@ def shift(self, **shifts): Parameters ---------- - **shifts : keyword arguments of the form {dim: offset} + shifts : Mapping with the form of {dim: offset} Integer offset to shift along each of the given dimensions. Positive offsets shift to the right; negative offsets shift to the left. + **shifts_kwargs: + The keyword arguments form of ``shifts``. + One of shifts or shifts_kwarg must be provided. Returns ------- @@ -2012,8 +2029,8 @@ def shift(self, **shifts): Coordinates: * x (x) int64 0 1 2 """ - variable = self.variable.shift(**shifts) - return self._replace(variable) + ds = self._to_temp_dataset().shift(shifts=shifts, **shifts_kwargs) + return self._from_temp_dataset(ds) def roll(self, shifts=None, roll_coords=None, **shifts_kwargs): """Roll this array by an offset along one or more dimensions. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 37544aca372..597b681bf65 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1759,8 +1759,8 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): align """ indexers = alignment.reindex_like_indexers(self, other) - return self.reindex(method=method, copy=copy, tolerance=tolerance, - **indexers) + return self.reindex(indexers=indexers, method=method, copy=copy, + tolerance=tolerance) def reindex(self, indexers=None, method=None, tolerance=None, copy=True, **indexers_kwargs): @@ -1809,7 +1809,7 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, pandas.Index.get_indexer """ indexers = utils.either_dict_or_kwargs(indexers, indexers_kwargs, - 'reindex') + 'reindex') bad_dims = [d for d in indexers if d not in self.dims] if bad_dims: @@ -2144,22 +2144,26 @@ def expand_dims(self, dim, axis=None): return self._replace_vars_and_dims(variables, self._coord_names) - def set_index(self, append=False, inplace=False, **indexes): + def set_index(self, indexes=None, append=False, inplace=False, + **indexes_kwargs): """Set Dataset (multi-)indexes using one or more existing coordinates or variables. Parameters ---------- + indexes : {dim: index, ...} + Mapping from names matching dimensions and values given + by (lists of) the names of existing coordinates or variables to set + as new (multi-)index. append : bool, optional If True, append the supplied index(es) to the existing index(es). Otherwise replace the existing index(es) (default). inplace : bool, optional If True, set new index(es) in-place. Otherwise, return a new Dataset object. - **indexes : {dim: index, ...} - Keyword arguments with names matching dimensions and values given - by (lists of) the names of existing coordinates or variables to set - as new (multi-)index. + **indexes_kwargs: optional + The keyword arguments form of ``indexes``. + One of indexes or indexes_kwargs must be provided. Returns ------- @@ -2170,6 +2174,7 @@ def set_index(self, append=False, inplace=False, **indexes): -------- Dataset.reset_index """ + indexes = either_dict_or_kwargs(indexes, indexes_kwargs, 'set_index') variables, coord_names = merge_indexes(indexes, self._variables, self._coord_names, append=append) @@ -2206,18 +2211,22 @@ def reset_index(self, dims_or_levels, drop=False, inplace=False): return self._replace_vars_and_dims(variables, coord_names=coord_names, inplace=inplace) - def reorder_levels(self, inplace=False, **dim_order): + def reorder_levels(self, dim_order=None, inplace=False, + **dim_order_kwargs): """Rearrange index levels using input order. Parameters ---------- + dim_order : optional + Mapping from names matching dimensions and values given + by lists representing new level orders. Every given dimension + must have a multi-index. inplace : bool, optional If True, modify the dataset in-place. Otherwise, return a new DataArray object. - **dim_order : optional - Keyword arguments with names matching dimensions and values given - by lists representing new level orders. Every given dimension - must have a multi-index. + **dim_order_kwargs: optional + The keyword arguments form of ``dim_order``. + One of dim_order or dim_order_kwargs must be provided. Returns ------- @@ -2225,6 +2234,8 @@ def reorder_levels(self, inplace=False, **dim_order): Another dataset, with this dataset's data but replaced coordinates. """ + dim_order = either_dict_or_kwargs(dim_order, dim_order_kwargs, + 'reorder_levels') replace_variables = {} for dim, order in dim_order.items(): coord = self._variables[dim] @@ -2267,7 +2278,7 @@ def _stack_once(self, dims, new_dim): return self._replace_vars_and_dims(variables, coord_names) - def stack(self, **dimensions): + def stack(self, dimensions=None, **dimensions_kwargs): """ Stack any number of existing dimensions into a single new dimension. @@ -2276,9 +2287,12 @@ def stack(self, **dimensions): Parameters ---------- - **dimensions : keyword arguments of the form new_name=(dim1, dim2, ...) + dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they replace. + **dimensions_kwargs: + The keyword arguments form of ``dimensions``. + One of dimensions or dimensions_kwargs must be provided. Returns ------- @@ -2289,6 +2303,8 @@ def stack(self, **dimensions): -------- Dataset.unstack """ + dimensions = either_dict_or_kwargs(dimensions, dimensions_kwargs, + 'stack') result = self for new_dim, dims in dimensions.items(): result = result._stack_once(dims, new_dim) @@ -2329,7 +2345,7 @@ def unstack(self, dim): if index.equals(full_idx): obj = self else: - obj = self.reindex(copy=False, **{dim: full_idx}) + obj = self.reindex({dim: full_idx}, copy=False) new_dim_names = index.names new_dim_sizes = [lev.size for lev in index.levels] @@ -2339,7 +2355,7 @@ def unstack(self, dim): if name != dim: if dim in var.dims: new_dims = OrderedDict(zip(new_dim_names, new_dim_sizes)) - variables[name] = var.unstack(**{dim: new_dims}) + variables[name] = var.unstack({dim: new_dims}) else: variables[name] = var @@ -2578,7 +2594,7 @@ def dropna(self, dim, how='any', thresh=None, subset=None): else: raise TypeError('must specify how or thresh') - return self.isel(**{dim: mask}) + return self.isel({dim: mask}) def fillna(self, value): """Fill missing values in this object. @@ -2843,17 +2859,20 @@ def apply(self, func, keep_attrs=False, args=(), **kwargs): attrs = self.attrs if keep_attrs else None return type(self)(variables, attrs=attrs) - def assign(self, **kwargs): + def assign(self, variables=None, **variables_kwargs): """Assign new data variables to a Dataset, returning a new object with all the original variables in addition to the new ones. Parameters ---------- - kwargs : keyword, value pairs - keywords are the variables names. If the values are callable, they - are computed on the Dataset and assigned to new data variables. If - the values are not callable, (e.g. a DataArray, scalar, or array), - they are simply assigned. + variables : mapping, value pairs + Mapping from variables names to the new values. If the new values + are callable, they are computed on the Dataset and assigned to new + data variables. If the values are not callable, (e.g. a DataArray, + scalar, or array), they are simply assigned. + **variables_kwargs: + The keyword arguments form of ``variables``. + One of variables or variables_kwarg must be provided. Returns ------- @@ -2873,9 +2892,10 @@ def assign(self, **kwargs): -------- pandas.DataFrame.assign """ + variables = either_dict_or_kwargs(variables, variables_kwargs, 'assign') data = self.copy() # do all calculations first... - results = data._calc_assign_results(kwargs) + results = data._calc_assign_results(variables) # ... and then assign data.update(results) return data @@ -3310,7 +3330,7 @@ def diff(self, dim, n=1, label='upper'): else: return difference - def shift(self, **shifts): + def shift(self, shifts=None, **shifts_kwargs): """Shift this dataset by an offset along one or more dimensions. Only data variables are moved; coordinates stay in place. This is @@ -3318,10 +3338,13 @@ def shift(self, **shifts): Parameters ---------- - **shifts : keyword arguments of the form {dim: offset} + shifts : Mapping with the form of {dim: offset} Integer offset to shift along each of the given dimensions. Positive offsets shift to the right; negative offsets shift to the left. + **shifts_kwargs: + The keyword arguments form of ``shifts``. + One of shifts or shifts_kwarg must be provided. Returns ------- @@ -3345,6 +3368,7 @@ def shift(self, **shifts): Data variables: foo (x) object nan nan 'a' 'b' 'c' """ + shifts = either_dict_or_kwargs(shifts, shifts_kwargs, 'shift') invalid = [k for k in shifts if k not in self.dims] if invalid: raise ValueError("dimensions %r do not exist" % invalid) diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 24ed280b19e..883dbb34dff 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -44,7 +44,7 @@ class Rolling(object): _attributes = ['window', 'min_periods', 'center', 'dim'] - def __init__(self, obj, min_periods=None, center=False, **windows): + def __init__(self, obj, windows, min_periods=None, center=False): """ Moving window object. @@ -52,18 +52,18 @@ def __init__(self, obj, min_periods=None, center=False, **windows): ---------- obj : Dataset or DataArray Object to window. + windows : A mapping from a dimension name to window size + dim : str + Name of the dimension to create the rolling iterator + along (e.g., `time`). + window : int + Size of the moving window. min_periods : int, default None Minimum number of observations in window required to have a value (otherwise result is NA). The default, None, is equivalent to setting min_periods equal to the size of the window. center : boolean, default False Set the labels at the center of the window. - **windows : dim=window - dim : str - Name of the dimension to create the rolling iterator - along (e.g., `time`). - window : int - Size of the moving window. Returns ------- @@ -115,7 +115,7 @@ def __len__(self): class DataArrayRolling(Rolling): - def __init__(self, obj, min_periods=None, center=False, **windows): + def __init__(self, obj, windows, min_periods=None, center=False): """ Moving window object for DataArray. You should use DataArray.rolling() method to construct this object @@ -125,18 +125,18 @@ def __init__(self, obj, min_periods=None, center=False, **windows): ---------- obj : DataArray Object to window. + windows : A mapping from a dimension name to window size + dim : str + Name of the dimension to create the rolling iterator + along (e.g., `time`). + window : int + Size of the moving window. min_periods : int, default None Minimum number of observations in window required to have a value (otherwise result is NA). The default, None, is equivalent to setting min_periods equal to the size of the window. center : boolean, default False Set the labels at the center of the window. - **windows : dim=window - dim : str - Name of the dimension to create the rolling iterator - along (e.g., `time`). - window : int - Size of the moving window. Returns ------- @@ -149,8 +149,8 @@ def __init__(self, obj, min_periods=None, center=False, **windows): Dataset.rolling Dataset.groupby """ - super(DataArrayRolling, self).__init__(obj, min_periods=min_periods, - center=center, **windows) + super(DataArrayRolling, self).__init__( + obj, windows, min_periods=min_periods, center=center) self.window_labels = self.obj[self.dim] @@ -321,7 +321,7 @@ def wrapped_func(self, **kwargs): class DatasetRolling(Rolling): - def __init__(self, obj, min_periods=None, center=False, **windows): + def __init__(self, obj, windows, min_periods=None, center=False): """ Moving window object for Dataset. You should use Dataset.rolling() method to construct this object @@ -331,18 +331,18 @@ def __init__(self, obj, min_periods=None, center=False, **windows): ---------- obj : Dataset Object to window. + windows : A mapping from a dimension name to window size + dim : str + Name of the dimension to create the rolling iterator + along (e.g., `time`). + window : int + Size of the moving window. min_periods : int, default None Minimum number of observations in window required to have a value (otherwise result is NA). The default, None, is equivalent to setting min_periods equal to the size of the window. center : boolean, default False Set the labels at the center of the window. - **windows : dim=window - dim : str - Name of the dimension to create the rolling iterator - along (e.g., `time`). - window : int - Size of the moving window. Returns ------- @@ -355,8 +355,7 @@ def __init__(self, obj, min_periods=None, center=False, **windows): Dataset.groupby DataArray.groupby """ - super(DatasetRolling, self).__init__(obj, - min_periods, center, **windows) + super(DatasetRolling, self).__init__(obj, windows, min_periods, center) if self.dim not in self.obj.dims: raise KeyError(self.dim) # Keep each Rolling object as an OrderedDict @@ -364,8 +363,8 @@ def __init__(self, obj, min_periods=None, center=False, **windows): for key, da in self.obj.data_vars.items(): # keeps rollings only for the dataset depending on slf.dim if self.dim in da.dims: - self.rollings[key] = DataArrayRolling(da, min_periods, - center, **windows) + self.rollings[key] = DataArrayRolling( + da, windows, min_periods, center) def reduce(self, func, **kwargs): """Reduce the items in this group by applying `func` along some diff --git a/xarray/core/variable.py b/xarray/core/variable.py index d9772407b82..d82fd6fb7ea 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -877,7 +877,7 @@ def squeeze(self, dim=None): numpy.squeeze """ dims = common.get_squeeze_dims(self, dim) - return self.isel(**{d: 0 for d in dims}) + return self.isel({d: 0 for d in dims}) def _shift_one_dim(self, dim, count): axis = self.get_axis_num(dim) @@ -919,36 +919,46 @@ def _shift_one_dim(self, dim, count): return type(self)(self.dims, data, self._attrs, fastpath=True) - def shift(self, **shifts): + def shift(self, shifts=None, **shifts_kwargs): """ Return a new Variable with shifted data. Parameters ---------- - **shifts : keyword arguments of the form {dim: offset} + shifts : mapping of the form {dim: offset} Integer offset to shift along each of the given dimensions. Positive offsets shift to the right; negative offsets shift to the left. + **shifts_kwargs: + The keyword arguments form of ``shifts``. + One of shifts or shifts_kwarg must be provided. Returns ------- shifted : Variable Variable with the same dimensions and attributes but shifted data. """ + shifts = either_dict_or_kwargs(shifts, shifts_kwargs, 'shift') result = self for dim, count in shifts.items(): result = result._shift_one_dim(dim, count) return result - def pad_with_fill_value(self, fill_value=dtypes.NA, **pad_widths): + def pad_with_fill_value(self, pad_widths=None, fill_value=dtypes.NA, + **pad_widths_kwargs): """ Return a new Variable with paddings. Parameters ---------- - **pad_width: keyword arguments of the form {dim: (before, after)} + pad_width: Mapping of the form {dim: (before, after)} Number of values padded to the edges of each dimension. + **pad_widths_kwargs: + Keyword argument for pad_widths """ + pad_widths = either_dict_or_kwargs(pad_widths, pad_widths_kwargs, + 'pad') + if fill_value is dtypes.NA: # np.nan is passed dtype, fill_value = dtypes.maybe_promote(self.dtype) else: @@ -1009,22 +1019,27 @@ def _roll_one_dim(self, dim, count): return type(self)(self.dims, data, self._attrs, fastpath=True) - def roll(self, **shifts): + def roll(self, shifts=None, **shifts_kwargs): """ Return a new Variable with rolld data. Parameters ---------- - **shifts : keyword arguments of the form {dim: offset} + shifts : mapping of the form {dim: offset} Integer offset to roll along each of the given dimensions. Positive offsets roll to the right; negative offsets roll to the left. + **shifts_kwargs: + The keyword arguments form of ``shifts``. + One of shifts or shifts_kwarg must be provided. Returns ------- shifted : Variable Variable with the same dimensions and attributes but rolled data. """ + shifts = either_dict_or_kwargs(shifts, shifts_kwargs, 'roll') + result = self for dim, count in shifts.items(): result = result._roll_one_dim(dim, count) @@ -1142,7 +1157,7 @@ def _stack_once(self, dims, new_dim): return Variable(new_dims, new_data, self._attrs, self._encoding, fastpath=True) - def stack(self, **dimensions): + def stack(self, dimensions=None, **dimensions_kwargs): """ Stack any number of existing dimensions into a single new dimension. @@ -1151,9 +1166,12 @@ def stack(self, **dimensions): Parameters ---------- - **dimensions : keyword arguments of the form new_name=(dim1, dim2, ...) + dimensions : Mapping of form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they replace. + **dimensions_kwargs: + The keyword arguments form of ``dimensions``. + One of dimensions or dimensions_kwargs must be provided. Returns ------- @@ -1164,6 +1182,8 @@ def stack(self, **dimensions): -------- Variable.unstack """ + dimensions = either_dict_or_kwargs(dimensions, dimensions_kwargs, + 'stack') result = self for new_dim, dims in dimensions.items(): result = result._stack_once(dims, new_dim) @@ -1195,7 +1215,7 @@ def _unstack_once(self, dims, old_dim): return Variable(new_dims, new_data, self._attrs, self._encoding, fastpath=True) - def unstack(self, **dimensions): + def unstack(self, dimensions=None, **dimensions_kwargs): """ Unstack an existing dimension into multiple new dimensions. @@ -1204,9 +1224,12 @@ def unstack(self, **dimensions): Parameters ---------- - **dimensions : keyword arguments of the form old_dim={dim1: size1, ...} + dimensions : mapping of the form old_dim={dim1: size1, ...} Names of existing dimensions, and the new dimensions and sizes that they map to. + **dimensions_kwargs: + The keyword arguments form of ``dimensions``. + One of dimensions or dimensions_kwargs must be provided. Returns ------- @@ -1217,6 +1240,8 @@ def unstack(self, **dimensions): -------- Variable.stack """ + dimensions = either_dict_or_kwargs(dimensions, dimensions_kwargs, + 'unstack') result = self for old_dim, dims in dimensions.items(): result = result._unstack_once(dims, old_dim) From 8378d3af259d7d1907359fc087dd0a6ca7e5ef17 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Mon, 20 Aug 2018 10:13:15 +0900 Subject: [PATCH 200/282] [MAINT] Avoid using duck typing (#2372) * avoiding duck typing * flake8 * Restore unintended change * Restore original form of _call_possibly_missing_method --- xarray/core/alignment.py | 5 ++++- xarray/core/combine.py | 7 ++++--- xarray/core/dataarray.py | 2 +- xarray/core/dataset.py | 10 +++++++--- xarray/core/merge.py | 15 ++++++++++----- xarray/core/variable.py | 13 +++---------- xarray/tests/test_backends.py | 1 - xarray/tests/test_variable.py | 16 ---------------- 8 files changed, 29 insertions(+), 40 deletions(-) diff --git a/xarray/core/alignment.py b/xarray/core/alignment.py index b0d2a49c29f..f82ddef25ba 100644 --- a/xarray/core/alignment.py +++ b/xarray/core/alignment.py @@ -174,11 +174,14 @@ def deep_align(objects, join='inner', copy=True, indexes=None, This function is not public API. """ + from .dataarray import DataArray + from .dataset import Dataset + if indexes is None: indexes = {} def is_alignable(obj): - return hasattr(obj, 'indexes') and hasattr(obj, 'reindex') + return isinstance(obj, (DataArray, Dataset)) positions = [] keys = [] diff --git a/xarray/core/combine.py b/xarray/core/combine.py index 430f0e564d6..f0cc025dc7e 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -125,16 +125,17 @@ def _calc_concat_dim_coord(dim): Infer the dimension name and 1d coordinate variable (if appropriate) for concatenating along the new dimension. """ + from .dataarray import DataArray + if isinstance(dim, basestring): coord = None - elif not hasattr(dim, 'dims'): - # dim is not a DataArray or IndexVariable + elif not isinstance(dim, (DataArray, Variable)): dim_name = getattr(dim, 'name', None) if dim_name is None: dim_name = 'concat_dim' coord = IndexVariable(dim_name, dim) dim = dim_name - elif not hasattr(dim, 'name'): + elif not isinstance(dim, DataArray): coord = as_variable(dim).to_index_variable() dim, = coord.dims else: diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 359812f2cc3..373a6a4cc9e 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1870,7 +1870,7 @@ def _binary_op(f, reflexive=False, join=None, **ignored_kwargs): def func(self, other): if isinstance(other, (Dataset, groupby.GroupBy)): return NotImplemented - if hasattr(other, 'indexes'): + if isinstance(other, DataArray): align_type = (OPTIONS['arithmetic_join'] if join is None else join) self, other = align(self, other, join=align_type, copy=False) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 597b681bf65..3d02b382921 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -2264,7 +2264,7 @@ def _stack_once(self, dims, new_dim): # consider dropping levels that are unused? levels = [self.get_index(dim) for dim in dims] - if hasattr(pd, 'RangeIndex'): + if LooseVersion(pd.__version__) < LooseVersion('0.19.0'): # RangeIndex levels in a MultiIndex are broken for appending in # pandas before v0.19.0 levels = [pd.Int64Index(level) @@ -3175,10 +3175,12 @@ def func(self, *args, **kwargs): def _binary_op(f, reflexive=False, join=None): @functools.wraps(f) def func(self, other): + from .dataarray import DataArray + if isinstance(other, groupby.GroupBy): return NotImplemented align_type = OPTIONS['arithmetic_join'] if join is None else join - if hasattr(other, 'indexes'): + if isinstance(other, (DataArray, Dataset)): self, other = align(self, other, join=align_type, copy=False) g = f if not reflexive else lambda x, y: f(y, x) ds = self._calculate_binary_op(g, other, join=align_type) @@ -3190,12 +3192,14 @@ def func(self, other): def _inplace_binary_op(f): @functools.wraps(f) def func(self, other): + from .dataarray import DataArray + if isinstance(other, groupby.GroupBy): raise TypeError('in-place operations between a Dataset and ' 'a grouped object are not permitted') # we don't actually modify arrays in-place with in-place Dataset # arithmetic -- this lets us automatically align things - if hasattr(other, 'indexes'): + if isinstance(other, (DataArray, Dataset)): other = other.reindex_like(self, copy=False) g = ops.inplace_to_noninplace_op(f) ds = self._calculate_binary_op(g, other, inplace=True) diff --git a/xarray/core/merge.py b/xarray/core/merge.py index f823717a8af..984dd2fa204 100644 --- a/xarray/core/merge.py +++ b/xarray/core/merge.py @@ -190,10 +190,13 @@ def expand_variable_dicts(list_of_variable_dicts): an input's values. The values of each ordered dictionary are all xarray.Variable objects. """ + from .dataarray import DataArray + from .dataset import Dataset + var_dicts = [] for variables in list_of_variable_dicts: - if hasattr(variables, 'variables'): # duck-type Dataset + if isinstance(variables, Dataset): sanitized_vars = variables.variables else: # append coords to var_dicts before appending sanitized_vars, @@ -201,7 +204,7 @@ def expand_variable_dicts(list_of_variable_dicts): sanitized_vars = OrderedDict() for name, var in variables.items(): - if hasattr(var, '_coords'): # duck-type DataArray + if isinstance(var, DataArray): # use private API for speed coords = var._coords.copy() # explicitly overwritten variables should take precedence @@ -232,17 +235,19 @@ def determine_coords(list_of_variable_dicts): All variable found in the input should appear in either the set of coordinate or non-coordinate names. """ + from .dataarray import DataArray + from .dataset import Dataset + coord_names = set() noncoord_names = set() for variables in list_of_variable_dicts: - if hasattr(variables, 'coords') and hasattr(variables, 'data_vars'): - # duck-type Dataset + if isinstance(variables, Dataset): coord_names.update(variables.coords) noncoord_names.update(variables.data_vars) else: for name, var in variables.items(): - if hasattr(var, '_coords'): # duck-type DataArray + if isinstance(var, DataArray): coords = set(var._coords) # use private API for speed # explicitly overwritten variables should take precedence coords.discard(name) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index d82fd6fb7ea..33a093d0496 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -64,22 +64,15 @@ def as_variable(obj, name=None): The newly created variable. """ + from .dataarray import DataArray + # TODO: consider extending this method to automatically handle Iris and - # pandas objects. - if hasattr(obj, 'variable'): + if isinstance(obj, DataArray): # extract the primary Variable from DataArrays obj = obj.variable if isinstance(obj, Variable): obj = obj.copy(deep=False) - elif hasattr(obj, 'dims') and (hasattr(obj, 'data') or - hasattr(obj, 'values')): - obj_data = getattr(obj, 'data', None) - if obj_data is None: - obj_data = getattr(obj, 'values') - obj = Variable(obj.dims, obj_data, - getattr(obj, 'attrs', None), - getattr(obj, 'encoding', None)) elif isinstance(obj, tuple): try: obj = Variable(*obj) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index e6de50b9dd2..3801225299f 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1404,7 +1404,6 @@ def test_chunk_encoding_with_dask(self): with self.roundtrip(ds_chunk4) as actual: self.assertEqual((4,), actual['var1'].encoding['chunks']) - # TODO: remove this failure once syncronized overlapping writes are # supported by xarray ds_chunk4['var1'].encoding.update({'chunks': 5}) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 3db5e6adc4b..a08f7262577 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1,7 +1,6 @@ from __future__ import absolute_import, division, print_function -from collections import namedtuple from copy import copy, deepcopy from datetime import datetime, timedelta from distutils.version import LooseVersion @@ -938,21 +937,6 @@ def test_as_variable(self): assert not isinstance(ds['x'], Variable) assert isinstance(as_variable(ds['x']), Variable) - FakeVariable = namedtuple('FakeVariable', 'values dims') - fake_xarray = FakeVariable(expected.values, expected.dims) - assert_identical(expected, as_variable(fake_xarray)) - - FakeVariable = namedtuple('FakeVariable', 'data dims') - fake_xarray = FakeVariable(expected.data, expected.dims) - assert_identical(expected, as_variable(fake_xarray)) - - FakeVariable = namedtuple('FakeVariable', - 'data values dims attrs encoding') - fake_xarray = FakeVariable(expected_extra.data, expected_extra.values, - expected_extra.dims, expected_extra.attrs, - expected_extra.encoding) - assert_identical(expected_extra, as_variable(fake_xarray)) - xarray_tuple = (expected_extra.dims, expected_extra.values, expected_extra.attrs, expected_extra.encoding) assert_identical(expected_extra, as_variable(xarray_tuple)) From 69086b332c6c950587830b266df4e624c2106d89 Mon Sep 17 00:00:00 2001 From: NotSqrt Date: Mon, 20 Aug 2018 18:31:15 +0200 Subject: [PATCH 201/282] Fix maybe_promote (#1953) * Fix maybe_promote With tests for every possible dtype: (numpy docs say `biufcmMOSUV` only) ``` for letter in string.ascii_letters: try: print(letter, np.dtype(letter)) except TypeError as exc: pass ``` * Check issubdtype of floating before timedelta64 In order to hit this branch more often * Improve maybe_promote test --- xarray/core/dtypes.py | 7 +++++-- xarray/tests/test_dtypes.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 7ad44472f06..a2f11728b4d 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -80,6 +80,11 @@ def maybe_promote(dtype): # N.B. these casting rules should match pandas if np.issubdtype(dtype, np.floating): fill_value = np.nan + elif np.issubdtype(dtype, np.timedelta64): + # See https://github.com/numpy/numpy/issues/10685 + # np.timedelta64 is a subclass of np.integer + # Check np.timedelta64 before np.integer + fill_value = np.timedelta64('NaT') elif np.issubdtype(dtype, np.integer): if dtype.itemsize <= 2: dtype = np.float32 @@ -90,8 +95,6 @@ def maybe_promote(dtype): fill_value = np.nan + np.nan * 1j elif np.issubdtype(dtype, np.datetime64): fill_value = np.datetime64('NaT') - elif np.issubdtype(dtype, np.timedelta64): - fill_value = np.timedelta64('NaT') else: dtype = object fill_value = np.nan diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 833df85f8af..292c60b4d05 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -50,3 +50,39 @@ def error(): def test_inf(obj): assert dtypes.INF > obj assert dtypes.NINF < obj + + +@pytest.mark.parametrize("kind, expected", [ + ('a', (np.dtype('O'), 'nan')), # dtype('S') + ('b', (np.float32, 'nan')), # dtype('int8') + ('B', (np.float32, 'nan')), # dtype('uint8') + ('c', (np.dtype('O'), 'nan')), # dtype('S1') + ('D', (np.complex128, '(nan+nanj)')), # dtype('complex128') + ('d', (np.float64, 'nan')), # dtype('float64') + ('e', (np.float16, 'nan')), # dtype('float16') + ('F', (np.complex64, '(nan+nanj)')), # dtype('complex64') + ('f', (np.float32, 'nan')), # dtype('float32') + ('h', (np.float32, 'nan')), # dtype('int16') + ('H', (np.float32, 'nan')), # dtype('uint16') + ('i', (np.float64, 'nan')), # dtype('int32') + ('I', (np.float64, 'nan')), # dtype('uint32') + ('l', (np.float64, 'nan')), # dtype('int64') + ('L', (np.float64, 'nan')), # dtype('uint64') + ('m', (np.timedelta64, 'NaT')), # dtype(' Date: Mon, 27 Aug 2018 18:21:19 -0700 Subject: [PATCH 202/282] =?UTF-8?q?BUG:=20modify=20behavior=20of=20Dataset?= =?UTF-8?q?.filter=5Fby=5Fattrs=20to=20match=20netCDF4.Data=E2=80=A6=20(#2?= =?UTF-8?q?322)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * BUG: modify behavior of Dataset.filter_by_attrs to match netCDF4.Dataset.get_variables_by_attributes * fix style and add more test for Dataset.filter_by_attrs * update what-new doc with fix for gh2315 --- doc/whats-new.rst | 6 ++++++ xarray/core/dataset.py | 19 ++++++++++++++----- xarray/tests/test_dataset.py | 20 ++++++++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4725fe74577..1ccf4dee00e 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -99,6 +99,12 @@ Bug fixes (:issue:`2341`) By `Keisuke Fujii `_. +- Fixed ``Dataset.filter_by_attrs()`` behavior not matching ``netCDF4.Dataset.get_variables_by_attributes()``. + When more than one ``key=value`` is passed into ``Dataset.filter_by_attrs()`` it will now return a Dataset with variables which pass + all the filters. + (:issue:`2315`) + By `Andrew Barna Date: Mon, 27 Aug 2018 18:48:59 -0700 Subject: [PATCH 203/282] fix typo in uri in the docs (#2386) --- doc/whats-new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 1ccf4dee00e..dd9eb9e48fe 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -103,7 +103,7 @@ Bug fixes When more than one ``key=value`` is passed into ``Dataset.filter_by_attrs()`` it will now return a Dataset with variables which pass all the filters. (:issue:`2315`) - By `Andrew Barna `_. .. _whats-new.0.10.8: From ecee9a0fe01db13bce1e234519614aeed53a7f07 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Tue, 28 Aug 2018 14:11:55 -0400 Subject: [PATCH 204/282] DOC: move xskillscore to 'Extend xarray capabilities' (#2387) --- doc/related-projects.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/related-projects.rst b/doc/related-projects.rst index 9b75d0e1b3e..714fbf98d7c 100644 --- a/doc/related-projects.rst +++ b/doc/related-projects.rst @@ -35,7 +35,6 @@ Geosciences - `xgcm `_: Extends the xarray data model to understand finite volume grid cells (common in General Circulation Models) and provides interpolation and difference operations for such grids. - `xmitgcm `_: a python package for reading `MITgcm `_ binary MDS files into xarray data structures. - `xshape `_: Tools for working with shapefiles, topographies, and polygons in xarray. -- `xskillscore `_: Metrics for verifying forecasts. Machine Learning ~~~~~~~~~~~~~~~~ @@ -52,6 +51,7 @@ Extend xarray capabilities - `xrft `_: Fourier transforms for xarray data. - `xr-scipy `_: A lightweight scipy wrapper for xarray. - `X-regression `_: Multiple linear regression from Statsmodels library coupled with Xarray library. +- `xskillscore `_: Metrics for verifying forecasts. - `xyzpy `_: Easily generate high dimensional data, including parallelization. Visualization From e5ae4088f3512eb805b13ea138087350b8180d69 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 31 Aug 2018 09:23:18 -0700 Subject: [PATCH 205/282] Mark test_equals_all_dtypes as xfail again (#2393) --- xarray/tests/test_variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index a08f7262577..904940cbbf6 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1651,7 +1651,7 @@ def test_getitem_1d_fancy(self): def test_equals_all_dtypes(self): import dask - if '0.18.2' <= LooseVersion(dask.__version__) < '0.18.3': + if '0.18.2' <= LooseVersion(dask.__version__) < '0.19.1': pytest.xfail('https://github.com/pydata/xarray/issues/2318') super(TestVariableWithDask, self).test_equals_all_dtypes() From a3ca579c3c6996a44440c7b0f5f68932b5a1c46d Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 4 Sep 2018 21:09:23 +0530 Subject: [PATCH 206/282] Silence some warnings. (#2328) * Make sure dask tests work with dask=0.16 * Silence some pnetcdf warnings. * fix sel_points, isel_points fancy indexing tests * Revert to using xr.ufuncs * Fix overflow/underflow warnings in interpolate_na These were being triggered by casting datetime64[ns] to float32. We now rescale the co-ordinate before interpolating, except for nearest-neighbour interpolation. The rescaling can change the nearest neighbour, and so is avoided in this case to maintain pandas compatibility. * Rescale datetime for interp() too. * Better rescaling. * Revert "Better rescaling." This reverts commit 76f988f594aea23d3acde1d603db5460c9010c1e. * Revert "Rescale datetime for interp() too." This reverts commit 9ac15ef677c4d21230b7aab40a65c1d7b0530ece. * Revert "Fix overflow/underflow warnings in interpolate_na" This reverts commit 1f1ec52707f8b2349461e41b68a7bc3918deb9f1. * Silence overflow/underflow/invalid value warnings. * Silence a bottleneck warning. * Revert "Silence a bottleneck warning." This reverts commit b9851275fdccd4c1cf8e662bffd5b1353b4ea048. * Dask: change from attribute check to version check. * Maybe this fixes python 2 failure? --- xarray/coding/times.py | 7 +++-- xarray/core/formatting.py | 2 +- xarray/core/missing.py | 19 +++++++----- xarray/plot/plot.py | 7 ++++- xarray/plot/utils.py | 4 ++- xarray/tests/test_backends.py | 49 ++++++++++++++----------------- xarray/tests/test_coding_times.py | 3 +- xarray/tests/test_dask.py | 14 +++++++-- xarray/tests/test_dataarray.py | 5 +++- xarray/tests/test_dataset.py | 4 +++ xarray/tests/test_missing.py | 16 +++++----- xarray/tests/test_plot.py | 10 +++++++ 12 files changed, 89 insertions(+), 51 deletions(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index d946e2ed378..6edbedce54c 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -183,8 +183,11 @@ def decode_cf_datetime(num_dates, units, calendar=None, # fixes: https://github.com/pydata/pandas/issues/14068 # these lines check if the the lowest or the highest value in dates # cause an OutOfBoundsDatetime (Overflow) error - pd.to_timedelta(flat_num_dates.min(), delta) + ref_date - pd.to_timedelta(flat_num_dates.max(), delta) + ref_date + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'invalid value encountered', + RuntimeWarning) + pd.to_timedelta(flat_num_dates.min(), delta) + ref_date + pd.to_timedelta(flat_num_dates.max(), delta) + ref_date # Cast input dates to integers of nanoseconds because `pd.to_datetime` # works much faster when dealing with integers diff --git a/xarray/core/formatting.py b/xarray/core/formatting.py index 65f3c91ca26..042c8c5324d 100644 --- a/xarray/core/formatting.py +++ b/xarray/core/formatting.py @@ -183,7 +183,7 @@ def format_items(x): day_part = (x[~pd.isnull(x)] .astype('timedelta64[D]') .astype('timedelta64[ns]')) - time_needed = x != day_part + time_needed = x[~pd.isnull(x)] != day_part day_needed = day_part != np.timedelta64(0, 'ns') if np.logical_not(day_needed).all(): timedelta_format = 'time' diff --git a/xarray/core/missing.py b/xarray/core/missing.py index 232fa185c07..90aa4ffaeda 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -3,6 +3,8 @@ from collections import Iterable from functools import partial +import warnings + import numpy as np import pandas as pd @@ -207,13 +209,16 @@ def interp_na(self, dim=None, use_coordinate=True, method='linear', limit=None, interp_class, kwargs = _get_interpolator(method, **kwargs) interpolator = partial(func_interpolate_na, interp_class, **kwargs) - arr = apply_ufunc(interpolator, index, self, - input_core_dims=[[dim], [dim]], - output_core_dims=[[dim]], - output_dtypes=[self.dtype], - dask='parallelized', - vectorize=True, - keep_attrs=True).transpose(*self.dims) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'overflow', RuntimeWarning) + warnings.filterwarnings('ignore', 'invalid value', RuntimeWarning) + arr = apply_ufunc(interpolator, index, self, + input_core_dims=[[dim], [dim]], + output_core_dims=[[dim]], + output_dtypes=[self.dtype], + dask='parallelized', + vectorize=True, + keep_attrs=True).transpose(*self.dims) if limit is not None: arr = arr.where(valids) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 179f41e9e42..0b3ab6f1bde 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -479,9 +479,11 @@ def line(self, *args, **kwargs): def _rescale_imshow_rgb(darray, vmin, vmax, robust): assert robust or vmin is not None or vmax is not None + # TODO: remove when min numpy version is bumped to 1.13 # There's a cyclic dependency via DataArray, so we can't import from # xarray.ufuncs in global scope. from xarray.ufuncs import maximum, minimum + # Calculate vmin and vmax automatically for `robust=True` if robust: if vmax is None: @@ -507,7 +509,10 @@ def _rescale_imshow_rgb(darray, vmin, vmax, robust): # After scaling, downcast to 32-bit float. This substantially reduces # memory usage after we hand `darray` off to matplotlib. darray = ((darray.astype('f8') - vmin) / (vmax - vmin)).astype('f4') - return minimum(maximum(darray, 0), 1) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'xarray.ufuncs', + PendingDeprecationWarning) + return minimum(maximum(darray, 0), 1) def _plot2d(plotfunc): diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 1ddb02352be..6221bfe9153 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -213,8 +213,10 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, # Handle discrete levels if levels is not None: if is_scalar(levels): - if user_minmax or levels == 1: + if user_minmax: levels = np.linspace(vmin, vmax, levels) + elif levels == 1: + levels = np.asarray([(vmin + vmax) / 2]) else: # N in MaxNLocator refers to bins, not ticks ticker = mpl.ticker.MaxNLocator(levels - 1) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 3801225299f..8b469761ccd 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1789,6 +1789,7 @@ def create_store(self): with create_tmp_file() as tmp_file: yield backends.H5NetCDFStore(tmp_file, 'w') + @pytest.mark.filterwarnings('ignore:complex dtypes are supported by h5py') def test_complex(self): expected = Dataset({'x': ('y', np.ones(5) + 1j * np.ones(5))}) with self.roundtrip(expected) as actual: @@ -2527,6 +2528,7 @@ class PyNioTestAutocloseTrue(PyNioTest): @requires_pseudonetcdf +@pytest.mark.filterwarnings('ignore:IOAPI_ISPH is assumed to be 6370000') class PseudoNetCDFFormatTest(TestCase): autoclose = True @@ -2658,14 +2660,11 @@ def test_uamiv_format_read(self): """ Open a CAMx file and test data variables """ - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=UserWarning, - message=('IOAPI_ISPH is assumed to be ' + - '6370000.; consistent with WRF')) - camxfile = open_example_dataset('example.uamiv', - engine='pseudonetcdf', - autoclose=True, - backend_kwargs={'format': 'uamiv'}) + + camxfile = open_example_dataset('example.uamiv', + engine='pseudonetcdf', + autoclose=True, + backend_kwargs={'format': 'uamiv'}) data = np.arange(20, dtype='f').reshape(1, 1, 4, 5) expected = xr.Variable(('TSTEP', 'LAY', 'ROW', 'COL'), data, dict(units='ppm', long_name='O3'.ljust(16), @@ -2687,17 +2686,14 @@ def test_uamiv_format_mfread(self): """ Open a CAMx file and test data variables """ - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=UserWarning, - message=('IOAPI_ISPH is assumed to be ' + - '6370000.; consistent with WRF')) - camxfile = open_example_mfdataset( - ['example.uamiv', - 'example.uamiv'], - engine='pseudonetcdf', - autoclose=True, - concat_dim='TSTEP', - backend_kwargs={'format': 'uamiv'}) + + camxfile = open_example_mfdataset( + ['example.uamiv', + 'example.uamiv'], + engine='pseudonetcdf', + autoclose=True, + concat_dim='TSTEP', + backend_kwargs={'format': 'uamiv'}) data1 = np.arange(20, dtype='f').reshape(1, 1, 4, 5) data = np.concatenate([data1] * 2, axis=0) @@ -2720,19 +2716,18 @@ def test_uamiv_format_mfread(self): def test_uamiv_format_write(self): fmtkw = {'format': 'uamiv'} - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=UserWarning, - message=('IOAPI_ISPH is assumed to be ' + - '6370000.; consistent with WRF')) - expected = open_example_dataset('example.uamiv', - engine='pseudonetcdf', - autoclose=False, - backend_kwargs=fmtkw) + + expected = open_example_dataset('example.uamiv', + engine='pseudonetcdf', + autoclose=False, + backend_kwargs=fmtkw) with self.roundtrip(expected, save_kwargs=fmtkw, open_kwargs={'backend_kwargs': fmtkw}) as actual: assert_identical(expected, actual) + expected.close() + def save(self, dataset, path, **save_kwargs): import PseudoNetCDF as pnc pncf = pnc.PseudoNetCDFFile() diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index e763af4984c..7d3a4930b44 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -538,7 +538,8 @@ def test_cf_datetime_nan(num_dates, units, expected_list): with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'All-NaN') actual = coding.times.decode_cf_datetime(num_dates, units) - expected = np.array(expected_list, dtype='datetime64[ns]') + # use pandas because numpy will deprecate timezone-aware conversions + expected = pd.to_datetime(expected_list) assert_array_equal(expected, actual) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index f6c47cce8d8..6ca83ab73ab 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -2,6 +2,7 @@ import pickle from textwrap import dedent +from distutils.version import LooseVersion import numpy as np import pandas as pd @@ -24,8 +25,12 @@ class DaskTestCase(TestCase): def assertLazyAnd(self, expected, actual, test): - with dask.set_options(get=dask.get): + + with (dask.config.set(get=dask.get) + if LooseVersion(dask.__version__) >= LooseVersion('0.18.0') + else dask.set_options(get=dask.get)): test(actual, expected) + if isinstance(actual, Dataset): for k, v in actual.variables.items(): if k in actual.dims: @@ -196,11 +201,13 @@ def test_missing_methods(self): except NotImplementedError as err: assert 'dask' in str(err) + @pytest.mark.filterwarnings('ignore::PendingDeprecationWarning') def test_univariate_ufunc(self): u = self.eager_var v = self.lazy_var self.assertLazyAndAllClose(np.sin(u), xu.sin(v)) + @pytest.mark.filterwarnings('ignore::PendingDeprecationWarning') def test_bivariate_ufunc(self): u = self.eager_var v = self.lazy_var @@ -421,6 +428,7 @@ def duplicate_and_merge(array): actual = duplicate_and_merge(self.lazy_array) self.assertLazyAndEqual(expected, actual) + @pytest.mark.filterwarnings('ignore::PendingDeprecationWarning') def test_ufuncs(self): u = self.eager_array v = self.lazy_array @@ -821,7 +829,9 @@ def test_basic_compute(): dask.multiprocessing.get, dask.local.get_sync, None]: - with dask.set_options(get=get): + with (dask.config.set(get=get) + if LooseVersion(dask.__version__) >= LooseVersion('0.18.0') + else dask.set_options(get=get)): ds.compute() ds.foo.compute() ds.foo.variable.compute() diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 3619688d091..5d20a6cfec3 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -672,6 +672,7 @@ def test_isel_types(self): assert_identical(da.isel(x=np.array([0], dtype="int64")), da.isel(x=np.array([0]))) + @pytest.mark.filterwarnings('ignore::DeprecationWarning') def test_isel_fancy(self): shape = (10, 7, 6) np_array = np.random.random(shape) @@ -845,6 +846,7 @@ def test_isel_drop(self): selected = data.isel(x=0, drop=False) assert_identical(expected, selected) + @pytest.mark.filterwarnings("ignore:Dataset.isel_points") def test_isel_points(self): shape = (10, 5, 6) np_array = np.random.random(shape) @@ -1237,6 +1239,7 @@ def test_reindex_like_no_index(self): ValueError, 'different size for unlabeled'): foo.reindex_like(bar) + @pytest.mark.filterwarnings('ignore:Indexer has dimensions') def test_reindex_regressions(self): # regression test for #279 expected = DataArray(np.random.randn(5), coords=[("time", range(5))]) @@ -1286,7 +1289,7 @@ def test_swap_dims(self): def test_expand_dims_error(self): array = DataArray(np.random.randn(3, 4), dims=['x', 'dim_0'], - coords={'x': np.linspace(0.0, 1.0, 3.0)}, + coords={'x': np.linspace(0.0, 1.0, 3)}, attrs={'key': 'entry'}) with raises_regex(ValueError, 'dim should be str or'): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 101de9fe8c7..068b445c69f 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1240,6 +1240,7 @@ def test_isel_drop(self): selected = data.isel(x=0, drop=False) assert_identical(expected, selected) + @pytest.mark.filterwarnings("ignore:Dataset.isel_points") def test_isel_points(self): data = create_test_data() @@ -1317,6 +1318,8 @@ def test_isel_points(self): dim2=stations['dim2s'], dim=np.array([4, 5, 6])) + @pytest.mark.filterwarnings("ignore:Dataset.sel_points") + @pytest.mark.filterwarnings("ignore:Dataset.isel_points") def test_sel_points(self): data = create_test_data() @@ -1347,6 +1350,7 @@ def test_sel_points(self): with pytest.raises(KeyError): data.sel_points(x=[2.5], y=[2.0], method='pad', tolerance=1e-3) + @pytest.mark.filterwarnings('ignore::DeprecationWarning') def test_sel_fancy(self): data = create_test_data() diff --git a/xarray/tests/test_missing.py b/xarray/tests/test_missing.py index 5c7e384c789..47224e55473 100644 --- a/xarray/tests/test_missing.py +++ b/xarray/tests/test_missing.py @@ -93,14 +93,14 @@ def test_interpolate_pd_compat(): @requires_scipy -def test_scipy_methods_function(): - for method in ['barycentric', 'krog', 'pchip', 'spline', 'akima']: - kwargs = {} - # Note: Pandas does some wacky things with these methods and the full - # integration tests wont work. - da, _ = make_interpolate_example_data((25, 25), 0.4, non_uniform=True) - actual = da.interpolate_na(method=method, dim='time', **kwargs) - assert (da.count('time') <= actual.count('time')).all() +@pytest.mark.parametrize('method', ['barycentric', 'krog', + 'pchip', 'spline', 'akima']) +def test_scipy_methods_function(method): + # Note: Pandas does some wacky things with these methods and the full + # integration tests wont work. + da, _ = make_interpolate_example_data((25, 25), 0.4, non_uniform=True) + actual = da.interpolate_na(method=method, dim='time') + assert (da.count('time') <= actual.count('time')).all() @requires_scipy diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 4e5ea8fc623..e7caf3d6ca2 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -267,6 +267,7 @@ def test_datetime_dimension(self): assert ax.has_data() @pytest.mark.slow + @pytest.mark.filterwarnings('ignore:tight_layout cannot') def test_convenient_facetgrid(self): a = easy_array((10, 15, 4)) d = DataArray(a, dims=['y', 'x', 'z']) @@ -328,6 +329,7 @@ def test_plot_size(self): self.darray.plot(aspect=1) @pytest.mark.slow + @pytest.mark.filterwarnings('ignore:tight_layout cannot') def test_convenient_facetgrid_4d(self): a = easy_array((10, 15, 2, 3)) d = DataArray(a, dims=['y', 'x', 'columns', 'rows']) @@ -775,10 +777,13 @@ def test_plot_nans(self): clim2 = self.plotfunc(x2).get_clim() assert clim1 == clim2 + @pytest.mark.filterwarnings('ignore::UserWarning') + @pytest.mark.filterwarnings('ignore:invalid value encountered') def test_can_plot_all_nans(self): # regression test for issue #1780 self.plotfunc(DataArray(np.full((2, 2), np.nan))) + @pytest.mark.filterwarnings('ignore: Attempting to set') def test_can_plot_axis_size_one(self): if self.plotfunc.__name__ not in ('contour', 'contourf'): self.plotfunc(DataArray(np.ones((1, 1)))) @@ -970,6 +975,7 @@ def test_2d_function_and_method_signature_same(self): del func_sig['darray'] assert func_sig == method_sig + @pytest.mark.filterwarnings('ignore:tight_layout cannot') def test_convenient_facetgrid(self): a = easy_array((10, 15, 4)) d = DataArray(a, dims=['y', 'x', 'z']) @@ -1001,6 +1007,7 @@ def test_convenient_facetgrid(self): else: assert '' == ax.get_xlabel() + @pytest.mark.filterwarnings('ignore:tight_layout cannot') def test_convenient_facetgrid_4d(self): a = easy_array((10, 15, 2, 3)) d = DataArray(a, dims=['y', 'x', 'columns', 'rows']) @@ -1279,6 +1286,7 @@ def test_imshow_rgb_values_in_valid_range(self): assert out.dtype == np.uint8 assert (out[..., :3] == da.values).all() # Compare without added alpha + @pytest.mark.filterwarnings('ignore:Several dimensions of this array') def test_regression_rgb_imshow_dim_size_one(self): # Regression: https://github.com/pydata/xarray/issues/1966 da = DataArray(easy_array((1, 3, 3), start=0.0, stop=1.0)) @@ -1511,6 +1519,7 @@ def test_facetgrid_polar(self): sharey=False) +@pytest.mark.filterwarnings('ignore:tight_layout cannot') class TestFacetGrid4d(PlotTestCase): def setUp(self): a = easy_array((10, 15, 3, 2)) @@ -1538,6 +1547,7 @@ def test_default_labels(self): assert substring_in_axes(label, ax) +@pytest.mark.filterwarnings('ignore:tight_layout cannot') class TestFacetedLinePlots(PlotTestCase): def setUp(self): self.darray = DataArray(np.random.randn(10, 6, 3, 4), From fc9ef81dbda163348316a9014bc44e7dae93a5ed Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Wed, 5 Sep 2018 17:17:22 +0200 Subject: [PATCH 207/282] add options for nondivergent and divergent cmap (#2397) * add options for nondivergent and divergent cmap * Update test_plot.py * renamed cmap options * Stickler fix * Another sticker fix * Additional tests and credits * Fix merge error in whats-new.rst * Fixed explicit cmap test * Update docstring Specify that colormap need not be matplotlib colormap * update docstring * keep stickler happy --- doc/whats-new.rst | 7 ++++++- xarray/core/options.py | 11 ++++++++++- xarray/plot/utils.py | 5 +++-- xarray/tests/test_plot.py | 16 ++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index dd9eb9e48fe..97794125665 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,11 @@ Documentation Enhancements ~~~~~~~~~~~~ +- Default colormap for sequential and divergent data can now be set via + :py:func:`~xarray.set_options()` + (:issue:`2394`) + By `Julius Busecke `_. + - min_count option is newly supported in :py:meth:`~xarray.DataArray.sum`, :py:meth:`~xarray.DataArray.prod` and :py:meth:`~xarray.Dataset.sum`, and :py:meth:`~xarray.Dataset.prod`. @@ -84,7 +89,7 @@ Bug fixes - Tests can be run in parallel with pytest-xdist By `Tony Tung `_. -- Follow up the renamings in dask; from dask.ghost to dask.overlap +- Follow up the renamings in dask; from dask.ghost to dask.overlap By `Keisuke Fujii `_. - Now raises a ValueError when there is a conflict between dimension names and diff --git a/xarray/core/options.py b/xarray/core/options.py index 48d4567fc99..a6118f02ed3 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -3,7 +3,9 @@ OPTIONS = { 'display_width': 80, 'arithmetic_join': 'inner', - 'enable_cftimeindex': False + 'enable_cftimeindex': False, + 'cmap_sequential': 'viridis', + 'cmap_divergent': 'RdBu_r', } @@ -19,6 +21,13 @@ class set_options(object): - ``enable_cftimeindex``: flag to enable using a ``CFTimeIndex`` for time indexes with non-standard calendars or dates outside the Timestamp-valid range. Default: ``False``. + - ``cmap_sequential``: colormap to use for nondivergent data plots. + Default: ``viridis``. If string, must be matplotlib built-in colormap. + Can also be a Colormap object (e.g. mpl.cm.magma) + - ``cmap_divergent``: colormap to use for divergent data plots. + Default: ``RdBu_r``. If string, must be matplotlib built-in colormap. + Can also be a Colormap object (e.g. mpl.cm.magma) + You can use ``set_options`` either as a context manager: diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 6221bfe9153..9af0624dbfc 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -7,6 +7,7 @@ from ..core.pycompat import basestring from ..core.utils import is_scalar +from ..core.options import OPTIONS ROBUST_PERCENTILE = 2.0 @@ -206,9 +207,9 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, # Choose default colormaps if not provided if cmap is None: if divergent: - cmap = "RdBu_r" + cmap = OPTIONS['cmap_divergent'] else: - cmap = "viridis" + cmap = OPTIONS['cmap_sequential'] # Handle discrete levels if levels is not None: diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index e7caf3d6ca2..c38ffeff884 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd +import xarray as xr import pytest import xarray.plot as xplt @@ -472,6 +473,21 @@ def test_center(self): assert cmap_params['levels'] is None assert cmap_params['norm'] is None + def test_cmap_sequential_option(self): + with xr.set_options(cmap_sequential='magma'): + cmap_params = _determine_cmap_params(self.data) + assert cmap_params['cmap'] == 'magma' + + def test_cmap_sequential_explicit_option(self): + with xr.set_options(cmap_sequential=mpl.cm.magma): + cmap_params = _determine_cmap_params(self.data) + assert cmap_params['cmap'] == mpl.cm.magma + + def test_cmap_divergent_option(self): + with xr.set_options(cmap_divergent='magma'): + cmap_params = _determine_cmap_params(self.data, center=0.5) + assert cmap_params['cmap'] == 'magma' + def test_nan_inf_are_ignored(self): cmap_params1 = _determine_cmap_params(self.data) data = self.data From 795a7bf26b6b4a6558c13b64864c4b5e0ea79016 Mon Sep 17 00:00:00 2001 From: Stephane Raynaud Date: Wed, 5 Sep 2018 17:18:45 +0200 Subject: [PATCH 208/282] BUG: Fix cdms2 related tests (#2332) (#2400) * Add curvilinear grid support to to_cdms2 and fix mask bug * fix indentation in to_cdms2 * Add generic unstructured grid support to _from_cdms2 and to_cdms2 * Fix indentation in from_cdms2 * Fix indentation in from_cdms2 * Split cdms2 unit tests and use OrderedDict * BUG: Fix #2332 about cdms2 tests * BUG: Fix #2332 about cdms2 tests --- xarray/tests/test_dataarray.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 5d20a6cfec3..29ddd40ce25 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2918,7 +2918,6 @@ def test_to_masked_array(self): ma = da.to_masked_array() assert len(ma.mask) == N - @pytest.mark.xfail # GH:2332 TODO fix this in upstream? def test_to_and_from_cdms2_classic(self): """Classic with 1D axes""" pytest.importorskip('cdms2') @@ -2931,7 +2930,7 @@ def test_to_and_from_cdms2_classic(self): expected_coords = [IndexVariable('distance', [-2, 2]), IndexVariable('time', [0, 1, 2])] actual = original.to_cdms2() - assert_array_equal(actual, original) + assert_array_equal(actual.asma(), original) assert actual.id == original.name self.assertItemsEqual(actual.getAxisIds(), original.dims) for axis, coord in zip(actual.getAxisList(), expected_coords): @@ -2953,7 +2952,6 @@ def test_to_and_from_cdms2_classic(self): assert_array_equal(original.coords[coord_name], back.coords[coord_name]) - @pytest.mark.xfail # GH:2332 TODO fix this in upstream? def test_to_and_from_cdms2_sgrid(self): """Curvilinear (structured) grid @@ -2971,8 +2969,10 @@ def test_to_and_from_cdms2_sgrid(self): name='sst') actual = original.to_cdms2() self.assertItemsEqual(actual.getAxisIds(), original.dims) - assert_array_equal(original.coords['lon'], actual.getLongitude()) - assert_array_equal(original.coords['lat'], actual.getLatitude()) + assert_array_equal(original.coords['lon'], + actual.getLongitude().asma()) + assert_array_equal(original.coords['lat'], + actual.getLatitude().asma()) back = from_cdms2(actual) self.assertItemsEqual(original.dims, back.dims) @@ -2980,7 +2980,6 @@ def test_to_and_from_cdms2_sgrid(self): assert_array_equal(original.coords['lat'], back.coords['lat']) assert_array_equal(original.coords['lon'], back.coords['lon']) - @pytest.mark.xfail # GH:2332 TODO fix this in upstream? def test_to_and_from_cdms2_ugrid(self): """Unstructured grid""" pytest.importorskip('cdms2') @@ -2992,8 +2991,10 @@ def test_to_and_from_cdms2_ugrid(self): coords={'lon': lon, 'lat': lat, 'cell': cell}) actual = original.to_cdms2() self.assertItemsEqual(actual.getAxisIds(), original.dims) - assert_array_equal(original.coords['lon'], actual.getLongitude()) - assert_array_equal(original.coords['lat'], actual.getLatitude()) + assert_array_equal(original.coords['lon'], + actual.getLongitude().getValue()) + assert_array_equal(original.coords['lat'], + actual.getLatitude().getValue()) back = from_cdms2(actual) self.assertItemsEqual(original.dims, back.dims) From 73f5b02a42a4003815d2bfc91e658195f5050be1 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Wed, 5 Sep 2018 11:19:06 -0400 Subject: [PATCH 209/282] Make `dim` optional on unstack (#2375) * Making dim an optional input for unstack method * Adding tests where dim is not explicitly set * Switched to using first MultiIndex in dim * Fixing too long line * Unstack along all MultiIndexes and accept *dims * Making dim accept interable or string or None * Responding to comments - if no multi-index, return object as is * Added section to whats-new * Pep8 and returing a copy rather than mutating original * linting * Adding back in error if non-MultiIndex is passed as arg * Reworking logic --- doc/whats-new.rst | 3 ++ xarray/core/dataarray.py | 30 ++++++++++++-- xarray/core/dataset.py | 74 +++++++++++++++++++++------------- xarray/tests/test_dataarray.py | 14 +++++++ xarray/tests/test_dataset.py | 9 +++-- 5 files changed, 94 insertions(+), 36 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 97794125665..98fb955ac48 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -73,6 +73,9 @@ Enhancements (:issue:`1875`) By `Andrew Huang `_. +- You can now call ``unstack`` without arguments to unstack every MultiIndex in a DataArray or Dataset. + By `Julia Signell `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 373a6a4cc9e..ae3758f0bbd 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1247,23 +1247,45 @@ def stack(self, dimensions=None, **dimensions_kwargs): ds = self._to_temp_dataset().stack(dimensions, **dimensions_kwargs) return self._from_temp_dataset(ds) - def unstack(self, dim): + def unstack(self, dim=None): """ - Unstack an existing dimension corresponding to a MultiIndex into + Unstack existing dimensions corresponding to MultiIndexes into multiple new dimensions. New dimensions will be added at the end. Parameters ---------- - dim : str - Name of the existing dimension to unstack. + dim : str or sequence of str, optional + Dimension(s) over which to unstack. By default unstacks all + MultiIndexes. Returns ------- unstacked : DataArray Array with unstacked data. + Examples + -------- + + >>> arr = DataArray(np.arange(6).reshape(2, 3), + ... coords=[('x', ['a', 'b']), ('y', [0, 1, 2])]) + >>> arr + + array([[0, 1, 2], + [3, 4, 5]]) + Coordinates: + * x (x) |S1 'a' 'b' + * y (y) int64 0 1 2 + >>> stacked = arr.stack(z=('x', 'y')) + >>> stacked.indexes['z'] + MultiIndex(levels=[[u'a', u'b'], [0, 1, 2]], + labels=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]], + names=[u'x', u'y']) + >>> roundtripped = stacked.unstack() + >>> arr.identical(roundtripped) + True + See also -------- DataArray.stack diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 06b258ae261..e98495e71fb 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -2310,35 +2310,8 @@ def stack(self, dimensions=None, **dimensions_kwargs): result = result._stack_once(dims, new_dim) return result - def unstack(self, dim): - """ - Unstack an existing dimension corresponding to a MultiIndex into - multiple new dimensions. - - New dimensions will be added at the end. - - Parameters - ---------- - dim : str - Name of the existing dimension to unstack. - - Returns - ------- - unstacked : Dataset - Dataset with unstacked data. - - See also - -------- - Dataset.stack - """ - if dim not in self.dims: - raise ValueError('invalid dimension: %s' % dim) - + def _unstack_once(self, dim): index = self.get_index(dim) - if not isinstance(index, pd.MultiIndex): - raise ValueError('cannot unstack a dimension that does not have ' - 'a MultiIndex') - full_idx = pd.MultiIndex.from_product(index.levels, names=index.names) # take a shortcut in case the MultiIndex was not modified. @@ -2366,6 +2339,51 @@ def unstack(self, dim): return self._replace_vars_and_dims(variables, coord_names) + def unstack(self, dim=None): + """ + Unstack existing dimensions corresponding to MultiIndexes into + multiple new dimensions. + + New dimensions will be added at the end. + + Parameters + ---------- + dim : str or sequence of str, optional + Dimension(s) over which to unstack. By default unstacks all + MultiIndexes. + + Returns + ------- + unstacked : Dataset + Dataset with unstacked data. + + See also + -------- + Dataset.stack + """ + + if dim is None: + dims = [d for d in self.dims if isinstance(self.get_index(d), + pd.MultiIndex)] + else: + dims = [dim] if isinstance(dim, basestring) else dim + + missing_dims = [d for d in dims if d not in self.dims] + if missing_dims: + raise ValueError('Dataset does not contain the dimensions: %s' + % missing_dims) + + non_multi_dims = [d for d in dims if not + isinstance(self.get_index(d), pd.MultiIndex)] + if non_multi_dims: + raise ValueError('cannot unstack dimensions that do not ' + 'have a MultiIndex: %s' % non_multi_dims) + + result = self.copy(deep=False) + for dim in dims: + result = result._unstack_once(dim) + return result + def update(self, other, inplace=True): """Update this dataset's variables with those from another dataset. diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 29ddd40ce25..a4562894583 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1663,9 +1663,23 @@ def test_dataset_math(self): def test_stack_unstack(self): orig = DataArray([[0, 1], [2, 3]], dims=['x', 'y'], attrs={'foo': 2}) + assert_identical(orig, orig.unstack()) + actual = orig.stack(z=['x', 'y']).unstack('z').drop(['x', 'y']) assert_identical(orig, actual) + dims = ['a', 'b', 'c', 'd', 'e'] + orig = xr.DataArray(np.random.rand(1, 2, 3, 2, 1), dims=dims) + stacked = orig.stack(ab=['a', 'b'], cd=['c', 'd']) + + unstacked = stacked.unstack(['ab', 'cd']) + roundtripped = unstacked.drop(['a', 'b', 'c', 'd']).transpose(*dims) + assert_identical(orig, roundtripped) + + unstacked = stacked.unstack() + roundtripped = unstacked.drop(['a', 'b', 'c', 'd']).transpose(*dims) + assert_identical(orig, roundtripped) + def test_stack_unstack_decreasing_coordinate(self): # regression test for GH980 orig = DataArray(np.random.rand(3, 4), dims=('y', 'x'), diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 068b445c69f..d22d8470dc6 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2107,14 +2107,15 @@ def test_unstack(self): expected = Dataset({'b': (('x', 'y'), [[0, 1], [2, 3]]), 'x': [0, 1], 'y': ['a', 'b']}) - actual = ds.unstack('z') - assert_identical(actual, expected) + for dim in ['z', ['z'], None]: + actual = ds.unstack(dim) + assert_identical(actual, expected) def test_unstack_errors(self): ds = Dataset({'x': [1, 2, 3]}) - with raises_regex(ValueError, 'invalid dimension'): + with raises_regex(ValueError, 'does not contain the dimensions'): ds.unstack('foo') - with raises_regex(ValueError, 'does not have a MultiIndex'): + with raises_regex(ValueError, 'do not have a MultiIndex'): ds.unstack('x') def test_stack_unstack_fast(self): From 66a8f8dd7f5a2997ff614f3966d1951587915e7e Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 6 Sep 2018 09:20:37 +0530 Subject: [PATCH 210/282] plot.imshow now obeys 'origin' kwarg. (#2396) Fixes #2379 --- doc/whats-new.rst | 4 ++++ xarray/plot/plot.py | 15 +++++++++------ xarray/tests/test_plot.py | 11 +++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 98fb955ac48..881ae52cdeb 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -79,6 +79,10 @@ Enhancements Bug fixes ~~~~~~~~~ +- ``xarray.plot.imshow()`` correctly uses the ``origin`` argument. + (:issue:`2379`) + By `Deepak Cherian `_. + - Fixed ``DataArray.to_iris()`` failure while creating ``DimCoord`` by falling back to creating ``AuxCoord``. Fixed dependency on ``var_name`` attribute being set. diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 0b3ab6f1bde..10fca44b417 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -624,7 +624,7 @@ def _plot2d(plotfunc): @functools.wraps(plotfunc) def newplotfunc(darray, x=None, y=None, figsize=None, size=None, aspect=None, ax=None, row=None, col=None, - col_wrap=None, xincrease=True, yincrease=True, + col_wrap=None, xincrease=None, yincrease=None, add_colorbar=None, add_labels=True, vmin=None, vmax=None, cmap=None, center=None, robust=False, extend=None, levels=None, infer_intervals=None, colors=None, @@ -794,7 +794,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, @functools.wraps(newplotfunc) def plotmethod(_PlotMethods_obj, x=None, y=None, figsize=None, size=None, aspect=None, ax=None, row=None, col=None, col_wrap=None, - xincrease=True, yincrease=True, add_colorbar=None, + xincrease=None, yincrease=None, add_colorbar=None, add_labels=True, vmin=None, vmax=None, cmap=None, colors=None, center=None, robust=False, extend=None, levels=None, infer_intervals=None, subplot_kws=None, @@ -862,10 +862,8 @@ def imshow(x, y, z, ax, **kwargs): left, right = x[0] - xstep, x[-1] + xstep bottom, top = y[-1] + ystep, y[0] - ystep - defaults = {'extent': [left, right, bottom, top], - 'origin': 'upper', - 'interpolation': 'nearest', - } + defaults = {'origin': 'upper', + 'interpolation': 'nearest'} if not hasattr(ax, 'projection'): # not for cartopy geoaxes @@ -874,6 +872,11 @@ def imshow(x, y, z, ax, **kwargs): # Allow user to override these defaults defaults.update(kwargs) + if defaults['origin'] == 'upper': + defaults['extent'] = [left, right, bottom, top] + else: + defaults['extent'] = [left, right, top, bottom] + if z.ndim == 3: # matplotlib imshow uses black for missing data, but Xarray makes # missing data transparent. We therefore add an alpha channel if diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index c38ffeff884..15cb6af5fb1 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1308,6 +1308,17 @@ def test_regression_rgb_imshow_dim_size_one(self): da = DataArray(easy_array((1, 3, 3), start=0.0, stop=1.0)) da.plot.imshow() + def test_imshow_origin_kwarg(self): + da = DataArray(easy_array((5, 5, 3), start=-0.6, stop=1.4)) + da.plot.imshow(origin='upper') + assert plt.xlim()[0] < 0 + assert plt.ylim()[1] < 0 + + plt.clf() + da.plot.imshow(origin='lower') + assert plt.xlim()[0] < 0 + assert plt.ylim()[0] < 0 + class TestFacetGrid(PlotTestCase): def setUp(self): From 59bf7a7c13d8b01fd9600cb76c82b35b465f5707 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 10 Sep 2018 19:14:17 -0700 Subject: [PATCH 211/282] add some blurbs about numfocus sponsorship to docs (#2403) * add some blurbs about numfocus sponsorship to docs * add numfocus to history blurb --- README.rst | 20 ++++++++++++++++++-- doc/index.rst | 24 +++++++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 94beea1dba4..7355848eca4 100644 --- a/README.rst +++ b/README.rst @@ -15,6 +15,8 @@ xarray: N-D labeled arrays and datasets :target: https://zenodo.org/badge/latestdoi/13221727 .. image:: http://img.shields.io/badge/benchmarked%20by-asv-green.svg?style=flat :target: http://pandas.pydata.org/speed/xarray/ +.. image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A + :target: http://numfocus.org **xarray** (formerly **xray**) is an open source project and Python package that aims to bring the labeled data power of pandas_ to the physical sciences, by providing @@ -103,20 +105,34 @@ Get in touch .. _mailing list: https://groups.google.com/forum/#!forum/xarray .. _on GitHub: http://github.com/pydata/xarray +NumFOCUS +-------- + +.. image:: https://numfocus.org/wp-content/uploads/2017/07/NumFocus_LRG.png + :scale: 50 % + +Xarray is a fiscally sponsored project of NumFOCUS, a nonprofit dedicated +to supporting the open source scientific computing community. If you like +Xarray and want to support our mission, please consider making a +[donation](https://www.flipcause.com/secure/cause_pdetails/MjE3OQ==) +to support our efforts. + History ------- xarray is an evolution of an internal tool developed at `The Climate Corporation`__. It was originally written by Climate Corp researchers Stephan Hoyer, Alex Kleeman and Eugene Brevdo and was released as open source in -May 2014. The project was renamed from "xray" in January 2016. +May 2014. The project was renamed from "xray" in January 2016. Xarray became a +fiscally sponsored project of NumFOCUS_ in August 2018. __ http://climate.com/ +.. _NumFOCUS: https://numfocus.org License ------- -Copyright 2014-2017, xarray Developers +Copyright 2014-2018, xarray Developers Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/doc/index.rst b/doc/index.rst index e66c448f780..6c1a8519507 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -120,12 +120,17 @@ Get in touch .. _mailing list: https://groups.google.com/forum/#!forum/xarray .. _on GitHub: http://github.com/pydata/xarray -License -------- +NumFOCUS +-------- -xarray is available under the open source `Apache License`__. +.. image:: _static/numfocus_logo.png + :scale: 50 % -__ http://www.apache.org/licenses/LICENSE-2.0.html +Xarray is a fiscally sponsored project of NumFOCUS, a nonprofit dedicated +to supporting the open source scientific computing community. If you like +Xarray and want to support our mission, please consider making a +[donation](https://www.flipcause.com/secure/cause_pdetails/MjE3OQ==) +to support our efforts. History ------- @@ -133,6 +138,15 @@ History xarray is an evolution of an internal tool developed at `The Climate Corporation`__. It was originally written by Climate Corp researchers Stephan Hoyer, Alex Kleeman and Eugene Brevdo and was released as open source in -May 2014. The project was renamed from "xray" in January 2016. +May 2014. The project was renamed from "xray" in January 2016. Xarray became a +fiscally sponsored project of NumFOCUS_ in August 2018. __ http://climate.com/ +.. _NumFOCUS: https://numfocus.org + +License +------- + +xarray is available under the open source `Apache License`__. + +__ http://www.apache.org/licenses/LICENSE-2.0.html From 4de8dbc3b1de461c0c9d3b002e55d60b46d2e6d2 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 10 Sep 2018 22:13:50 -0700 Subject: [PATCH 212/282] Numfocus (#2409) * add some blurbs about numfocus sponsorship to docs * add numfocus to history blurb * add missing logo file * markdown to rst in readme --- README.rst | 7 ++++--- doc/_static/numfocus_logo.png | Bin 0 -> 24992 bytes 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 doc/_static/numfocus_logo.png diff --git a/README.rst b/README.rst index 7355848eca4..12650b1db1b 100644 --- a/README.rst +++ b/README.rst @@ -109,14 +109,15 @@ NumFOCUS -------- .. image:: https://numfocus.org/wp-content/uploads/2017/07/NumFocus_LRG.png - :scale: 50 % + :scale: 25 % Xarray is a fiscally sponsored project of NumFOCUS, a nonprofit dedicated to supporting the open source scientific computing community. If you like -Xarray and want to support our mission, please consider making a -[donation](https://www.flipcause.com/secure/cause_pdetails/MjE3OQ==) +Xarray and want to support our mission, please consider making a donation_ to support our efforts. +.. _donation: https://www.flipcause.com/secure/cause_pdetails/MjE3OQ== + History ------- diff --git a/doc/_static/numfocus_logo.png b/doc/_static/numfocus_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..af3c84209e06d9d4b106be987ca0c0013ff697a2 GIT binary patch literal 24992 zcmdSAq{|T!lB1NC96jmLAq~=? zAaZY?@9%zi{{irYwltaY%b&fH05?Bmth|5p>U(`Ek=W;vbWT$}OJJWr}J=2y3NL?%p~5SU}|msfRR z7+d7Sh$f2qmDhFFdX`zg2&yCh&kheMDv8TmBTZOlt+)J;q7mGT{tvhKkTGDRh|kWl z##>6r#7Jvqf6c8|NCR*)d4)vXCw-<4mj5%L18vp%vbkhkHHIAdmSTbTv=So;u8lOm zGi6C?MYwtu@sCf|&`Bz4%cGiPqVs>2H&*BvIqui?~6bAe?yadCaTrRwk|piB^NCX;LI}&X_%T zlZqxcnF+D(yCdu+_@I3!=Mj~6V#(hqa;x&MbSaxZz_1h950tM?gWQ{~9&Hgm#2Uf> zCJ)%@1_I|GTW`Umlqyy${`yWw{AwmEp#obLdRykmJP+b3MV%E)xK8A(X;!q!+6y7K ztuBs}24+ReEit5MAr$%in{*M6Q5&gkAl$ z=G#Vgop;H({N^o2WXKn(2X&T)2WteC`w=Y6>>_p1n0~Nbh{?VKTQBU3>Mcojbx(oC*<1PQ+^lWEgjj9m#TYLSP2iO77H{I!&@J{4 zG6*i9Z2wwLFyjuZ!5^69MESQ-9x%tx{sOKs4(%`Gb}9qc_DBh-FtOpq2Ni&7=^tK5 zUUypE%-_>zx6OhpI;5|Swzx@W8&WtQe+e}u4D*KDaYxz&b|4Y}dFhD1HuKJwUcAOTR$pw+C341VpL(Xdy?x(cOa!#WJ$Z1Xu%x zustH~iwyO9sTzpuA8R>ntmb0*$8)+)34N`VLRG#3(X3B+S-u?dtDCY&3Ib3(#uVr= z+iI|7c(Q23rzj2b%GGi}igg2VT_a`1aJYL<1m|7b0#@!=c~vt-I$_ zy(Vpp=r;R5+0SWJq^VKbWM&lK_!`G?Df?O7LWA+Rp;ItioD zA}lt&2zl7bxD4xZ9vI8a*3V+^lNyLO2pHu@1l1C%pFbY-ZBhwsdr4q@{8n&-;=}O9 zNKCehJ%2P5dZGj(#(v`@%Y;{nsWq&srFwC}?M0KAO*$biKx>Nw`GE1A)@P<} zmQkjmrk#{k=FzpU(_#(VF`RjGe6iA`=tp!YO3A%}zg#0jTJ3*Ct5ArGuYxk(E(9oI z4l*bg&cn}zVta1uN4_$MHEAg=y{Mc2i#SA)iqUb}hT2Cd#KSasD!{RC;t0^|@! z8!Gt*1<2O?6)C0j9wWs4>;oLx@Vir?>L^~VKOL|I7R@UzEMF6#*oYvkTNuf1bec>!a~O`SV8a zK>Z$hh1W}YwU65*oc<6(`A8v`Bj)W@BiXUAM+sMY<{3_CQMALNC>5f@3?X3kWbiPB z-WnLmOwD-73Cxm2c&oeUA>;Vwt^NvavI_WF>F&bfG0EE>nv;|thr`Ka;FrO1TU^2{ zhmVjCQKYpJ?^N#fm5zNrJ|PQ(gt*88cR4%tFF!ZAnT^hKUgT6M{c-VR=gXq1DknzS z;glsqEscFHc}gycXsNl84)hIid89T)QWNL-YvOL|ig5jycMCG#GZW4!2RBr5+v>Jw zaCq^FYgQCW;$Y?7>>9D0n;J|-Sqs3vQ)~Dcv){cm(!$v&6g#-|BPNmgAVtoth5-94 zz+M2MLIJdJ2hMe7<$LU>q_osrN2n5`H^`t?pJe#+zxL<-?#?krrPmrHmjtotj{UPI zKDepG9M-*Q$(FVXC0!7bQPp$GYSIUV$H}z5!3?#{efM^HB@K9n@ z!GFPH^W#H1{Cwc);tZ4cA_tJ8)$uze)QY~@Cq(n8Y}leM`K__;?#FG*FYJ=y>fcEi z)Rxk+E5+o}Q&|H(jPZLr!}4;<}5M^#KhPke|rKH zGaagM{5UgK`?yzPsPDlh7gWedNYYE@06H8;4rG<%%EkrWVwWD6n*0{e#rPddjB)nfAw=#W_Je$$1okiJkJEGSS7#U5neUcC+NC?2aY( z%vng$QN-A>Zm0g+f{$w9$G^E7i!JWEtbot|<_D1f4{UQeHa=ncNq19`|qzT zN-jw+jthIitmg+FZ3pF%w=@B$jMD)^=odn$*3C$ksR_=a{HG2fz9VG?z06j}fZJ*ng7DD=}d&wrXIGT_BYl#rx|gpV4q!SC ztu+iGE*tn-R7QYxKr$V#;uEPg(wg$H4NHoprZr~>W8>=kt>5u|>-FN+5B+}nFGnL& z$I=^Q1cSedIzKeH!^8MTvC_PTvBMh+^Km^ZFjK#lv=zRX@Xvu)S)p?$?+ij*?jceH zA9rc^+4WOh(Fw`&_TCEQc0LFF#1Ii4S!WzHQmZ#8;L3N1%VdbywhFQheuz)CAdMDD zH4yxrK-Iuk`k1b06Z5<3fCI6fV-@CV>H`nu4^lLRNqyH#yukB$X-Su7L~X0|39rFF z`CAiTina=p1F#`Li&sXhB;{F>Nz@fb>G$fzyN?|~g&l-YikC}_nYm>6idd$WI=i<5 zxxjDv20Tg{Xd2?8Nblp-@wHnB>uGXiZPw*s`4K7m#O3I19LL8yVvqYhr-W=p>Kzu9 zCwWU_V!u2Kh%jr)UFC%D)mTy^>d7FbI^vve0%1G$yU)il-x8}qGujB&M6sNtGpGM6 z{E&78>#S_OerhW!N!~{unE}93cbWd#sEjAtcpMbrTVg8S{i@@O{K0A-E6~L4qGau~ zusRIp_idpeH}566?u6aC$as-Pp%42yvo!!g13(pl>g_{7lS9$(CyWEy8CcP|9UVu$ zQ?38w8G4a|mm$B9kTblNm0-QAh=ahNNAU#E&$8mV#yJD>BycAQq<1!uk59ulqNizk8DqZZLTVsyR~Ms>uQNi+ zF_UQ(x86=C)BUIxUuuqP-q%R25G`MB1&n&6)P_n9^LcCEn+rctAnV5}Jt12KU^{?w zxf*dUyKCt~j$~*Bbi(R)KRqNCa*oS+QCcX^E%rO7MSV*{8#>Q3j`0fXQvbBxKYEIc zS9$q^0M!q-6DHg88uajs)H@{T&P+K1gXt_%k2n(`v??>&xwR_<3AERf{o5I#{;2~e zhXCte?QshB%uBdvyoWwtVyq7`jFvSE%-yWc3 zKgKR=#HFWtu!<4Q&3u*=`1A|8_DQLG!-2;=HIN6wO0?tWmO;B1WA&C=!*WVait+yu zI^~Rbu=b^&z2lKX*)u=f)-D0afWX<+Gq5*!mk4SU8S0NbA@bX7Faqx_aRV&9**oyG|CVG7Zv|iO+1^0}j}M4X4v#7b-G9i(`pIS^&z90QDif z9F|UIe>B4i%o0N<(VmCU9lzhtO?9LOT6_S`rS5)gLD|6niWfD^3czqwMQQ@{OZKq0 zfdKvaS3HadIJ%uPvcX59KP-NxQP+P^OfNkB#Fbq9;dEA(+JigwOFiHX6~Sv>FLHv` zE380z(Xe+KB=y>JX6hvdwjj)h=|F_5n~o!uGzUPXE(zf_3-x(xL1u zf!c~soZd|R*OHJz#wfu{e?DG-obf@-0*APQ4tqg_P$RMs-N1LD!*33PT7a>0Ap5y= zP6le-PxM**mkhx^uhM)Up6*S2mLF4Tmd%*yWE5NSIDSKV2z3uw5>Xqw^sN0#JY;|W z$rdfLFOoa?^lq_Ib5GGxj}J8>o)Icp-vX-tQ%`R~{{U_M{N0btw!zED&?Y)bxcvaW zUiu-5h9y~lPwZ>_C)Jdq>Y~|Y<4UC~B`_;j-Y3fbi8`5N$ZA7t1ICr%JFtEmL=h$i zn`jcqcP!RW1x<&!ID>!Kvb|KRiF{0aG~lto)ID79a_|`HPGL6NGPUTOwf^dW^Jvbz z?ad?sPF2V;nl|z=h!@A zwad={RDXb-(y_DU=L2n`<;>v2TSRE?tmp5`W!aR_%kQ=@z}<<*!rivj7$KIkqcYe~ z+4GA&VzgbeHcMzY*kX%=TFlgmS`BmyJ?o(QXAlyv_TEJrXtDLKaDDt?#Nye}fREmo z3@M`G8NwSbfs>-HzsxELrgteZbu07dkxToy;g`QU?hpVpnG5rT!+ax9xjDV4!wwU3 z8(R8yJNdaq>a086`M^jV2h@uDxIj#~$V+yY8#H)rxR3|MPB~J*OvJT2@aTHQGx%EG zbAuKt{)7=)yN5?NL9~bo(O=xA7XHcnbKDO&)~Q-MJ2>V{NSQ)hI*H6hWepZ*UW>{5 z(MT%<_0dBg(6etMYrxU0yr6z%R1xRb1qJ!v>)fMSH|Jm{ggE?Z09C7tf#gqv1KS

lNDwKhd1%oqYm=8aa-2eeik$=;4_I#)^{qLFKVyXH z`ByT33k+Qv5qr;Zv|Cx5*$jZ_9enRtd=?D+dRlsayuim6RG4^9N5yp${EM|TrS ziyTvXL!8t=Uro=IjS^Fz-)s^<2~biYdv1fn1%<-8iuc1Oxy>SgyM(blk*Y_(uWPm7 zcunb0jQ2se#nzT?H=3GU(xzS<_5MR1_Yy*xxQEPJgLAM{FMT5lk$v5y&|0@b4>W}( z6u5glz}=fc@oZ-^2pRcohYU!Ch-ZV6SyY~)k1ph9PE+oon-o#Wwaa*4lA)g>#lQUV zv){&qBHj0Y`*1zhP9WL$Cn$G63XqxPRmYa|jly zZgQL;zP^?cyMRX=ZbZr~SI>a;y3x{F{`r7;`6Gn9k7(7xCICxJS~He(Mvv42)uc7J zm;5Cbg;N9x5pskYoA z1VhBsKytV|wU@U)c%`3S)PJ4~-eu8a;y)1M5~ikuDdU`_pOcn~cZL<-8C^jws3rH< za(y;xONEb#NB6NK@*oyq)^FgH)s|eB?^P#Ofd|M%s|F%Oh!YUh51P{K?s=BU;5QOS zq_m*8;4dc`4U@lBNc8VFkn)mAg_YNZ&mDLb%;58Xk_@j{ZKrO&W$|q5uF-`>);t|B zs{=Tt_*Kl6fhQnMKMPpu20*Q-o0Q3n2}F56irJdJ_}}_I2=V%w;mCjQ0S*@TH+*FC zo|ut+il=~D^+j8MjKZrB1l+A_9W!jtGfnNZPNinT^<^Q6r+?lE()(pTn*pGt=xhZJ z78_WaWAw=V|0$?}?jXb`=W?U^g2z1ekk-~-cs20b%n8Qn3mOFtmTpXK4kP4@d3@*v z&`lvP-zA!YQpei>s7VSGqiw^=cuUL9zoAvO^Hi8#X1O@5xgc zR~$x8*!x?;N&u`AE+YpjyZihyCl>6jndtF*y+L?a2vV<3xQR3D(jh5Ex+~6PYof}| z#vak281)gR=cqzF^WPzXIQPGez#MQ-|9x-d0O`ZC;qb_fOFz5j%uZ=F&SW4Ysm= zZ4MlisMU}P6---QXpI+{;v}2uw0P?6QLJ~J2UUtms$A0EGR_MPaLmH-P$sqxr;{F| zsDm@Z^b0~fH;M;eKz{z)Cqa;)KU1Jat|eM)0JUDT(+r$l3ycL06hR$Q>shHq2W8Nb zcp|7c9o)Xn9;(W`WR6wyA=4H>*AqNg-RRRa7~`bu4;6MP*Kbvs$DcK4b9&~0}nY)Tx5c365f z1`Zloe76J-yzi=k_<$C%a`U-qx;i6YmYy~m(BaTchaGBlPrK%_IpTgN$VFLv-MmfT zNoS-Cag*G&N(>P;nL7Bbe>~um(20XO67)VyC#e`DoJ4$Rm6xGiw0&!ROPnMu~$7kaOA%Mh7=}0bpfu z3n<#yrDy@${ z;}<@V*)P=p(<^aAP(TvYLBc@-vOr8!T+-u2pVJYUh?Dqm2QdH|$C@klJ>Ov@qxRR` z@X7ER_$36Q=zQoml9Xh!^pp%V|DCBP%90S44crY2@uq1(A?sx=TxG;=J*txiu-%eWCh?Fg_0EguF8X&>W@ zH-s2MGL&R;tEu3k$dH!kPMWhoZ+hyszgti!RGmyGrGbU;Iq3XFqYj z*?+ozz6C(iemffJr9sgSrMkBLkr^?A;Pw4v>ccb1AN%5KIOf_Nj%$8mLiY?IP1#LpzPJOGl-t*YjadbQCT@lxyPByM7~+3Cc|Eg2oj(e>FG}wxE#yH za9Odtl9KbwJ#RCjNh$fR)RBiy2PkHo0~bit&_vu!l^x1jeSe1u7g@t3=u{OOI9k0G zgW?oCDZ?xYpEg%XJ$K}ANG60#Z)DBjx(J*)YPcw#xFoCRy1d$MF1Pt@Wq~|$_s5BP zh!ZV-r&_X+2&YNI9d(m6WW@?o(^mT2YXz8TQU-0?%7iaA5?iP?l~SS#!gaVf>DE%t z2z!c!$;578>UlUI!cQGy(ug)Ozlysyg+Wu|Ax*CLY}A z_xIWosloS*gA9QK9qsK#AE_4BbrCLFyvJx!x4$M5;dD5mMX6fPHh@JBwO;56xtZS% z3YcSl0n+B^NFs*xD?KWG&z;cqUv}Xdb6OQI^;$*X6AG%jJzMkpNCxVU!M{0gT zD5aj@(1c=x?*a!=1SkeS`vZrP?2i%hpiQ7s+REYRZMmFQ>bOW^Y)YhHv{fc4T96tg za5&#}2yI}aJ$)s32OkebA?1l_sbIRPTTT2?ves+avM(SM|E;W6Q$VM07J$!BEO`s+ zEqQCkNBr7X5_yv zxBp@ZKMwaA!<9eWL}8F(RcGz@<@OFOF5D}ih_k~nMoeX|Q1mr3!BE@QPoI%6QxFZp zo1mvMyGw?#YR|GZff;B~1l-NkRAB%8UL-MuQdCKq$r_@8XuY>XJKZVznpsrT z%J%`zLsO27^>jQ=wnV4{DslEA`Gz}cptYP9Vo6qupGj6yPGmHQczURX49jWbed-k- zNmoUj(iSkF#VIB=nzQ__EHD31TP{UaIWIbaxh%e^D<$OR#obm^-~#SA3INIL*OiRY z;Beh@HweI`X}{O%d?-(6^X&EF>x5)qkS%_BQfQIC#ppe|(9Ke7y!7>9<-JPU#q06% zdrZdpUQx}_SUm(kdhyxzyZm2nR%%CvxEdNPgclylSYH)s3Sx1^WhauhWwN%B;NX=! zyzwMlB8vdZM~G$g;d5169HU>Cg=%P zRXh2lAHehufXfbXA%mN=&v86(3^w78T=7N)lVRh;zgt3dN*4PJ;pdww*%27`MzEShDw>!aiU0mI> z6e@3ize&EIFyxP=26Avii>kvOMykq{*EUA@`%mJdiwco*z#$rO@Rt(+6&73Rc-}=d zMhaoo-#Z|ZyFi#ujq;J=3N7j3K-*Rsb0iq!Mj@*O{io3VtNzj55khIhvce(dG<9=7I|F_nPnn{`HtnoGSh1HLv zF3~@e%>U^<8SGj(P870N6=}{xbi7^=2wGL+zlwk5pN;_ zRPK^hbO@P1_Q%<%DzVUV;#+Zt67HjBkQ^M3#5IDr5d$0C?)oK)0QH9eQd-N+e;QBH z;o`i8<0zzLkbapumf~?~t69rID2f2*Lqcd!d%O%`$lm=}=PG}EJjS$G%+LM@^-<}X z{WYkXK3v!s?UoC-EDOe66pp)QK5IpRfLWSXPn`&F;~bd?&%p?=oM~_uCI~Xo;Jl41 zaaY$cA=yFH`(NH5+}DF4`FWUV_5amUwECYsaNe!f8mXvTIGxWF_iR5*#V7~wX8GIG zsQb9^x-!0oW`8#WF~*>jz{`7O7K6Y zon4bglk3!r6*%Vf^Q(c_|0}Z`9*mPVPC~CvwBUmPRH+}A|H-G>bU$75D}fcXFMsIoX?23IJv`O(Ub^@20oyx`H_}{P5dJBJ<=j z`A@?!<6yqz8Ml&j%!er^f3q#6V8Mlxe{AefZExHG`31|FM}{YP*o(i?|LG;iPG#LF zR&Wv7L;#)S+TK=GQ(SVk!DH2OL*o*9)uXJoQ1qX?S_7ea1F?)YWf1JBjZQIK34SBd zDkX-_+~Gh@CTE&yB}DMHtVr#hT!rBV*f2h?_=iE5<4ygn1zeQPsHXx-l+-{MXa5M* zKw|nIQyWMrf?jyevIb1zLw^3Y8faytDUj_79CuNaKHJJke{438a2u+XHYTFZxD@&^ z#`c-!=r1}H?u@_z{MKxu4+hFQv;+pIs!cU!&r+!{*~WbHjSSuUE%N1p6p5SIGNjiH zy~cH6={dN$C<$PM5HVoH0V*1V*ad9Nw2*%_!+Qzm>nMi=jT zxiaEBqv5)E@1;3iJ5~JQNkTx#UTG>UM^K8djM@ngZ4li&O7ToB1T5!X#j6if6`w@$ zP@pC#?k+BTnu1H^!t#(km)ZNo|3$ydjB_Kc%NaP$yzOFLmr{~O(>;iGb!P0Y?V&{2 zAhCLb^;Yk@+-qOti!#SFU_&aMY`S9>BzO=n6LM`sc4de*bszKEqZCEl@xja($*=+e4 z`_tDvR^Df8aoYm<5HIX}xJ$oMV|EV#MWJ(GV3~Bc+78E?H~{hX)r@51ZEVg=7W?@h zA#!ieMvLh`zc|dfY$sBd|k?XnxsD}d&*m! zQ+zCYQ#2InvTd=HkodeFUOfJV8rn?N>Hv9E^%=whWrK31i}wsj2+987uX5=G&OO@l zNxahk_A759VXeA%h4agmSWGZ*0zZ)(RzatBO*p3>{F5R&4CEzcq^8}lx={~BT-Iwx zC>A5?L(~Gj-rcv!>~06v#Zq<_eVut2XBK(^)s4X*0o2X!d zUSBZ=EQ;^vG@EaopvN?WN-)UO(clOSNOr}zh&FkkrPV%_aLg1uIM>@NGJDmu2Wk9v z;jLSS16&shOhe=8J*ApFQeZ&Rx?C2Qt8cIJG;c@tqGOtf68#G}(nf4>+a%MH5PD7U z8lIOfgB9gaLXXHTEd&^Fk33IXbkxf`KfEuXMTWx+Q39yd;`ar(Z__Z)H`sl%h038- zw^!QOMDLvP@-Jiem{gsR8F!?D>6Y0z(^TN>RZ?o!-zTFA`zjiXk0>y@)XUDC#U$9I zVqYE!->hA0#)gEKm%Vefo7Fwi@10sjl=6K<@FK zms-t7yi5&)X)2xmS6^Etfu{-U*X>N~*#fS7?b=o$W+r-wlx5cl4E5lys?ktmcBn^E z+d*h5yujkOTY>=Y5`%VfesXw(4FEda8u^9t`e*WYTMcCE8YzULx^dI*-a_jl^G*g6 zqGGc^rgh-g-rdzRJPy70w|9o@Qw@5g4=m2Jca$@9zvGU`qC>w`P#@ibmMqNS;U zlOA&)Ad2oo-ZZunM)cGjBI*4RF!5q_f}I^lBs0oTm+{lS(>Fpjf6*^w%H0#GPDINZ z1cu|V53}6@l0Z@T1B$xd%aAD@P8${_w8x){om$?PlrTTE{{~^yF*n%NzeQUH-dy0v ziYs%V0&B=vTPiQ9cs4Xhw4+?IyOLv>)7N~K+xXN!no~@mT0s=>*TB*l#*#W*1Zp+` z9DufQVKXP!;h;Rm0HKgXpcE}RmQ~(piJ!~lG}Kv!5b@pBbj+C8QIrJk;)*s=rwES6 zX=R^(R^UbKi=Nid{oT)6A0ipkd_j}$8C}EuO6b+(K+46kwRKhyXxT@SMcJ@Kg z;@Zhw51a%9D1|MIu=vCcNd%TFZc)Qs#kliNEScHUF+F=o*bD;!?>V1UW-WP(UU;OS zx$3BLLX7U@+}&0U)5VA>M&NV1cdSlNfz7TV@5#rr*Eenr_SX-1ID|;&`_2H9i~&ah z$>DbmqpsTsR(rM5iYTw(M7X(+PdY}DH_7WU1**^}@Yh8oW)*UZ9|k@|I$F{r?&fP0Vx9)wwH6||DB?{h5y|Au-=U)UNTfK z)^n6#tCuRDQ?{mnRr#01%yBq8e)+GrB4qOvf%HxP4_@5|=bku$r@fH*j zR?D1F?dQ09(y`f&p;NWQDu>NGvgCbA^(%>g{)p@Th};u4tCenz=&dEop#INbo44&2W zEgC&x`dP`9V#LXDf``0txp2Y;60U4B(&Swpj zYtCMilN)1Y-AkN17%B15@?9(`F?GQmD?O&B=dFWq+yo+SCMr){4tG>2D+#)fa(_Fi zCC_%)JGQNo9F%**rYIsKl8C2FS@3(JG^_hFanh2uj(VF2*XJzq7V3+TG}hEQQrB~5 ziV}{Qx!b$I(X+YRopz5A4KEB48)1pVgc295piM7%VGD}iX_WMOmY6y47eNDEiFXU4 zwxH-v`8m-;DYXB)6#2Oyf=wjJznCmXr}VZ$L+mxx+PrtNR*orAV@ay(@2^#KaMS-x z4i9dNs;i`FA6phMHUFp&EAlEZw>cv*_x)akG#)xmLQL9Tlv2?ttH1C;VcK)}U{pl@ zBcUb))mAf}@>0azi(2k6T4K#lS4irFo&7sDuQTn4cjHE%alrH6Z|1aT%TpnC&CY#z z-VpH^UY30UPl)y~(>C4xV2`37e(9qa|R_L-{uFj;@eaes46&vDH%e-%W=5xmWHKx_VP|n!@X()5EVmH#cm@T#IWoudd^=CMdZ^8dZBd zftO}MNiRqx-WB=Zj4VThy=M#!0uXq1h`W9$d)Ld3K8MTW-lz2yquAVZYN854TUO}7 zG8scim$>~3F-2QDvGd0lE4_#VLqDE@v;a9*Qi~`R1kDM2ZbJXj;=W*1g15p+PaJ{# z@V7I4Ot=956AOWk5A{vI2TUV$Jw56h{8zJnUk?sC1y{XR312j{?7gAdw)%M1WOSYy zB3NJGpD>(r%On5l)&)(L+YJVrPcwPAWLPpfrtJJR?CL^+;Uj}R6bq6mB{`u+NmT8z zfXByiBh^|Lhb`XEezVEK{K&10cxR&9R%w-zMG|w`TV&ym*`i)9oYgcqX1>Abq|L4T zbl21W-OSSq!;Mnqt%)8sQAe7p6S}0h6Wzl`uzwNN;x}#VTDtY>-{Zgn8%<(O61uI# z7WXY5U97kA=Iup|u?G>fYqxi5z=;$=Lml*!waMKri&RAmexQ!utG=p7osL?%-PGV#LJ(u;-iyd@9;DNLI0qOHqG8txLtQ>Uj<3K>TdpZA|tqRqhjz5xpjZqS>OS4TV{eHTcoO!u3 zSt6d12%wAC|5}KaSVFJOp2(+_j#J{vur-!!oEWkVdGgvluvGZIVeA*ZNC{}k_QQ2+ zfVsc*gKx_ztKQSM6pgoFv`qJcSQP)-PT%)j#KRLaY3;2lUsLMb52e`~8!jW*ubR9c zntb@)l@6@b6S+39_rIWJx<4ZM>Qg~E&6gCFx1=N4zVO)q533C>Wv4$k9zwaY`6wTF zl`sse$E>d_OTooBYiAYq_aQj@YpU_}zzZh13<}{AeuOswRTtw-hHU>YqHhhHUjeqi z-wi(3LE=Djna?e{y03vw<5j>2!CbTk>0RX)5f}p8(-v#SOS$D%I!QGzOW{sDr=^U* zO6)z%;T*S>PPXI9TQ%$CyI;IDkgf7bh(~C)oBKXQ=dbGmo0;j)c>ZWgt0DIco1;=a zcV(rV2=9d2QqeBwKK|RtVUvrE+sYemtv>Nh<;=||LYrT+RmBHgIr6hDlX)+De^mnC zsmNBY;=Rlv?nSzYMSA~?o0A~qXr!Q2 zw|w;1Q6B1=8b>4iXi}8aVbJ$9mDSMlafpQP&Ym_{J=`ds$L zChJH&He26v!DK{t=Y27|C!CXYgNXf@#o>@FU{a~qr`uIw=ohDfV0hu#XAGu9$Ycy! zV*1W5vo=3Zfl9;5!PA-}Of2+;-P8;4zw?JO&`@}M)Du!mgi`g~2gZ8?@X{nNR zt(v-zMJ0;`zE+G5xhd$Z6Dj-mU?rI0X=$5*jqok9f2;nS*>SFoNoppPlE8glSy33y z87_{<@Celb@(x3ESG&SUgJ8d%Cm!%)Be8$TY6KJPgZMWG-v?**U|D=}T{X;Ub;V^$ z9v?4T4IZ^Y2A&?GFuz(}NFM2UNuN04am!B?(zcH5wx54KRd^ZPfc9vSW3S#FnwRvWPmZRA3mZy&67 zF@^b+R#R^z8bAyaH&-K$?$8_K2|-V0X>x zb>Cxj!a5>cfdW1x=#e|drJ(&#Cso;z{gVnp$kmAPeQ43l>D=$TlW!nSS+OYRpO(XY zCmR?W!*?&*6NNreZJb}N6L`Jm0r-pP;YHe^6}YcZaq)xGOwGMjBT|>XDNm*Tt=UGf z_hw;EZ#28|z&Gf0u~F0NeRVO-xw*wYzH2k)&^_`IU8HE7DI)+fN4zOX2-Tp0>SP<8 zxr_)RqN~HP$}8UxcD8e-iXE2|ygh&m)>-4NSJs9%a`}Oso^aHmM4#y|zB=fNTX;FA zy~|3ZWju19K$X(ZKO#hLg`1~3&GUikuWvdmbDuZplXN-t-oC@=%=0{pIrPIiB#aiR z{-af8gbU;%*01o|ct;hJ-NFqSy=MD!h0+l%sJSCv3c9BUnm0jxVl8?EudJfkWGTcu zZJUcR!eCR11<_MZqG68gs9hc1u7@vfT~Ju@c|~FVQCD6oEV&3S8~-?;TD|bsA3xXa zALkc=LiD+pl~e{+Im-rbt2ZzS>AijOp-HPK8vfWnAjE~Ah)tZ%mCvgP6U`Pr3&}?9 zI*>Lzj8)2zp)`z{d;wRP@@R-PR?zj54Is(eEWENo6}wTs7P;B<{p)(*@s1R3Ir{T; zxkR0s)U^S)kGXGtYR)6qIM!4pThEYhbXj!%yq)Dz-ALEQ4bs*j%I+w&T-9UU6Q!Jz z{F!UgQ^`EWROw&OHR0K}sv}L5{v)Fzk2^u5ut%( zhuT9h**hh#_~|pV`0%Ex&-5)DUA=N@srTTj_vDJuzkeIabrX}LSvAlrI-YgOg>146@DYKXLU$GNZ zPr7B)JKkgK5bCH9a~Ak_LwcxtdEVnO@^xD7j+1B!joxR9M`V=ZYX-WlqV?Iz10@Mp z^CO&nWg-PBSfd(w?$-?%z*FZJ#IiQf^mTQ=8$ zNxIsvWItg#O*b+mO}Jg5PiONYd!nX08=dzU1ec|?CTNIVYqj}W@qr<>{z~sxoKzcKYuilC->zha;OVOzn#Afw&9`?P`72b~irNKBY-PW+{T9G@r z*$AncdkWL8nU~pU(ygb9$$vjqlH3wIXZR|}dg%H|{LU%hv*oG1!q3_W<@;N!rQ9s_$vQ z(jzeu)4xICm`rdM6>Nj=GAlmPWZm(V>^GjJ;@r269z2slrnkz9UJF;YB6eM)@4pqz z?O(pWB8Q2nuwHKhFM{rn#2q*71M8A7+~8u$`1Ph%lPybRK_rOL9P{~&%B-O;5(&QY zocq@4-Chs@V%X8PcfzK=icFO+w-Y(>j!0$R`M%$^bNxG)iaefrrt(SaTG{TiQ-KP5 zO>b1>3%EK@Ym))CAS5U3$m8{rz?K}m$=NTk0hn?UiWSM_4p3wAj^uQlJ(XdBu7LgYm?em zp7O|8p2*>nQQRZ5bibAVg7O zTm73-Wgiy*=T6c~Hl~fIfd>>hck0|~%CDDEE#f~qd!NkOM@+OZm5quta(huxg$?vGB3VF+|`5x7*N#X`xpjeyijO7B>s|SpR@P< z?6n2Y$!`m@r-ay|!UMZEPYl?UyO+UCU37%!kIQxqm*&3gf1^fivIreaM{S+teq5O` z?j@#r%l>iD1-TR{NZEQvR@X@h`R21X`wz26`U;tS@fqAfqtr7xi%LSYv*Zg$}V`{6n1!`a9I6cxz8L zuHc6@&%dxe;~OP41?4_#?gL*@SM8r40_Mq5y)Vt2vhsbWn#CuNMAR{gg1_M@Lab*| zhb<2~JjCjfUjeZ9yurXz!i&dpQOR9ziCtccD{J|L?|q~nJ(%L7Mo~#Uf9Sm+F9!^; zdiPotcR`Uu>Xc-^Hg6;4ebas0smU8^yZiOmpgtug-}c~7UXmgQL2R?I!JlORU^(g! zrek*Jg5r-zfmeC-`zsoB2(F=!yEoU9wC^tyLy02WjNdn2`U+Pq4bS{|d8PUmmDZ4> zp236;=E~n2Aw~e zV{u>=8dZd*n#tv%(y>RT&4Pc*ADLio$6)y}fJEHAi>p$Abd#x{tlX3*QdN{UtN+!JILpMI@*^!{&MbHZS}5 z;y`0xz2W&r7Z^1lM(cPYdQuGx0I7i>u43YADqsGPm)5qVuj!qy{qFxVvtGGNoEgae zju!f0b3NyJrdiAVQSp)VkR$t{!cwTt^mU+`k9422A@z79yVX$6^9mC5$f|&DI~_k8 zLfm`r$|<$Egs4s1=~kaIPp|!P!sl|m^T5WbgecOnL0Q-u?k+{*y5j#+-I@PG_5T0g zBtzMfC5&a#D{Ha~! z`(J#2p4*w*IoEmKu5+E|^|;>;GH!Vt>!hqT+@-nxGSe6=UiBKuPOR#iv6lU``K+As zT_#~W6?{l6WnP2zSCf==4jiEut9K)Xo;pEYd6AK|*~6;H&20)td>E@XCt9_3^)lXt z*U_AXT@gDyy{vcq$)R6Yfl7j$So?CR82U`;9`fdfa9Zzkr)6`2aVhlP6_)*t9Lrx7Wpmp4FrOPmTZh?R z$+j!(@s9hNkN1%8)^`P2WDHX&j6K{d6TO0>tK8lMlgG;0; zw^^yrO{+CN=dlYr9d7p-!=0^;^gO<>E;|yjqK%w26Rpz&LpaPzbbar$BKkFBE9NVt zq~voSO-jq5H$PDLxi{S5;D|ITvEQ61>QiR@o8X_GX>8RKqArLJ>3TJiNr|iZ9@>#B zfU*2~zxJR$Ocp+N@l9pH;+gzp3Gsg)SUSIm#<=IJ#A>E~;>$^RY6{iZ_pte+1gNx8 zk2X+91DPZhE5t|tsFgb%a;jmL&LWje)j8x?3fn>D_TqCaJP6gNXnGxfOk{`M3L{p| zhR!iC|CjD8UGpGfCC<(UV;`O+-g6I{?S5bS0G)qb+x%YBJn{2 z?;4*;I)x3nZ~0*K=hU9J-$e#M*S|8Shqt$zBhNYr_gw|ACi?6{p%9+m#7tznI*OKq z0u7j}qdQM#&Mai7SFwyR9>N)_4>(- z9Q(B~BDxBiG(5(`bHfkf?2QTcYC_L6Stm)ku`)L^q*JQbg+m2W2l8(|!SnM0Y9KLq z{?(C;Gy_R-V@`2MXUzyX)Ag)7)@FnT9(3*KaD`b(SCrR;QL0W9_fAJzS8%o=meQCD z^N&LfGvCWO9@=7gxWDf17e=E6hn2DKLM-}2YPpMAQNyqsVd8=1yVx=GOo7q_Ee0X$ zUuaty)l02c8NV8VHlw_$S9If_kz*T(y7A`q3EIrGBqz`!WoB#6r8EMqn5i;->>uSC z9;7*B&J*GK6=}c<6skzJ$tRcuOZ}}GSByp)e7Q1}FZ8gp$S+56gk=9R2nfS#+f+vj zEADY~HrELE$4G?kN0+4A(~&(_zcFIealbXxp->39kxR9(jwDxt$&gW(ci z>h@x}m5$Pf=}B`;7~LWbT6qfDOJ*maT4K!y{zC0w2TLzL+Iw99UgYc%hLEs*ygB}P za=)9C%J*WBsTdAU7u=>3+}!6eBY*oe_d`ydn#FZxzIHZcrMSYt=8}Cg*=7@c851x4 zTg+;KW}H)P;xHryZdpr##KzwhL^wSIXo2jpSQ($~jx}LAz?yr@v@ZVxQ$HbY=%mVU zpvE}=he?}9!hW~p?@@Ab0a-G%xZ`rF0!e3oxf3>K;?)B_T`xqrF2b7hAlWboW}6P^ z$7#hd*Yo27g+kjmV~+{h-M1buEf1%fInK#`j5nNBNcFBL1Z;|3rQJ)4UM62(U(?}) z0_)0;O?LlXNVV}_T6aG_{!<6!T{aQj-Fi7xl`NECFON_i**>3WL|+zT=P1rBI@WO> z2ohVHO1gI(K%IpVXL;Gob)1h0w@VfG+9}z#LuZR-EcZy^Q=%zcI z;4mWy83X>s2oQROkX72WZil-*Mpy0PCgeA>y~0;!(~w0KTu&{JYa{qB`-wRgS)|V_ zUeZ&dHA5xRuCY?!Qmm8y{QLx>n_OO4qs`E^&p(?n?W6rs3gwkp>-1f@0oo?FbX}YaN+>R4^_vDsG|I`p&Kl6s4qo zaYtdt)(7i{qN$cKf822a1Ngv>)1j)>x{K(lGMn7$m6ucUUTD;wDD#aE;l+i_U~w>D z&S2aaUG3Zd_|Dk!`+4G_`?WAtBO3D4w(?T-E6KO70LaQlZ6kQy^SM?YW4zk!3a>5! zO!w$(7EI{|X&wYDEYrq+3BSBbaQd!w+m}A}KW4ya9B|$ibK|q%_@@y`UGmk`yV-l& z{uxffWoqe)zMO^4^TBfuIb|hDTQ+L7;^#>vMGNZrRwD&8cxp@Pu%}{>kxT4*$#yF? z4vYcQ1)y-~#{>bjeT_+9-bY-Of;O^d|*GrPrV zaWKQL-;}dZ!I)n8R5R^+yxlXLo2tIzF>$~Ldn3?Vsu5YvNxxXFyXkEG$)+I{H_2KO zzGG=1)|lw{zZ$%*y=^~yEZ#odBe|0*JfXwd?*YNIM7oOTl{9H+tC&LYG3S3=TmCTk zuws0P?U84uP|1!ZQdJ`~u|C{YGAh$jfyP3C#CyFn;TexjVU~mjUzWdKBphMr1J`)+ zx;84aL4Rh8B@q4{if-E0IZwuiq;p}^j#h>g@@x##w_e!mD1R+(KQ`OAWYja<=KBc> z(qknypY^yg^etrM3Q%EYKmpQe#R%Nn+b#=^bTuzXQrKDvpb6tE z>{(3yGxaPK7#Ry@a*)KGxDW9#5ybI1S?1ifPLyU`9~3wm{0t+zgf&afa_HzHFk(B zyja}!g$MD86nXT*vID)8zLm!kn!Pg~!UnENXLrlzDgUtVPIK5S^J7o5g}abT+Zli+ z4pugXNdXgh(5)DihU^2RO3pdpTbC6*aLem(kX{V&a5ieCm+a zo?_qBrdNay3ZDnM34fkJU2^Sy)ZiP}^(KFl^%F`f&Xbi1$!4Y^XETm`M&Q~;Vn46e z&$H^=qLT{f>%g6;{|rBf=oS8Wg-94czQ2F&OT!kefmWdZQ%178->huW<`|xrjud^6 zo(Uy4@u*-aP=~OG|Iv^pyIAt0tgNIP%7EAMq-aqeaJ4f1D{rzH#^M5^ZH$0o+7B72 zRaGOoMJT=cuc==$G@!jfaIy9x97gvKe%>AM0@3>TL|n5$QV^>uG?2Y}+QYBqd&LVH z3-wYfgQmT4L(OTx3Mdima#sP@5~4F$FUL zCQ0j{CqnU`4~v|vB;}&6SGRPTopiGWmZ$Z|*?4l=`h)G!=Wi!Jztp4iCBw#AxLqB= z2|Is#;#obIf7HcR!uAEJ^3FD3O_nDQJfh6+n(}uzgQBezvkCu-k-3#$KCGwL{xFF> zK;i<#%O3d&1usD=*!_LH;thhj5Twg}rKN|8M?MHA_RE-ij^e=e;`V(|5;ZyVD`RK= z+WT^HQ7p)`&HHjT4RS$sji6Dt_qqE$h*Mj>tABXk&2NO)Dg1+V_L@`rJ*>a4E=uWk zKV3&>24LnTvWz-&{rL8g=F@8ZffXSDS8?KwW!sLWn!QdFHd`X+krcBp@%=E(`c}{7 zKJ`^d1x_|a;!~Xl$`6o3-pL5UqW~>Nirzrf2D7qR7vF-rWHkG8OPF-^KV_ASEHI)(F=V=S0%{7zaG!6 za>mflUZ--g+@T>s5t)jgzkTDvaPrrRGykx@c&YPG1jCrOW7xIjr}{F{$qj(d9k=u? z(o$l+WG^HzGZ#q@J!ipZ2s>l@GXHqCq~kM3@-Bri{#nLjM?9KC)NIr06V#gft7!p$ z*q;_Y-yjccwQx}1)g4oZKyPEPMfmDeSC5!H)j?V75{);G)#N^GCC%4$bR{~X`Upt;- z#i5ukbKx;*rP`Sh$LkCLql;Y@j-F%&C8b~Z2blEM>C8S+8E%&2q9+v^5q^iSIHeIA z_TbBMo`>NpIRfcPN~paFtMFpV>-X_Oaq|}y`l`>h>%>A{Lyta#nKv48zt8x`MRgUe z)l}?mb^Hp7yFJGFhO!L>P0_#Lf8}o({ZK*D|G<}=mH@jseVjtE5z4GwRylIOBOpE^ z6dt5;7f(fPj#-0&O9;?G4-$!JdUz3y#oHLC@OA{-Ez;if34RYg_NaQ^$#3^jR57J_ zmKJbqKM$-bKeb5J{^~P~R-B>gyo(5Z`RASEqufa-`~wUxIru7w9C}&M=#qA4HMGY_ zCS7jeU98%I8#k z)!74nf^pz*1i@hBim$51Rw$N*O{rs_Lxq!M)Ku-=7QS-wNmzmK*-?uDZ>krUpHQVB zx{Z!~XXV|zdJg|N4&oTpcHcK4d3o?mo__*3g|CM z{+zG9O!NNUrCt?ZtQ1XKV@|Ut7~bPRHb(DC#!Ms?iUl zinEnh|0?wk_&!n(wO$g`X>7QJs#>{SIFOzFiXF2oFMaf4J87w&<8!Yr!eHI}WKT8G z;Z%T^b>2-M*p+GQ3OW|6Kd$_%IG81A5mn5wRUR#zGJQ9LkV8WT%a{P%7c8U*xu74T zs^oc)8ryC+c>)NoF2{zF>S1j#n<4NLI)HROqrXt{gu;4N%{eWn{)W;$WKO7~haPpO z;0_)eGIsVrspmf0&j}j3`|hvxoQp*LlyPwV6b+e=SxJpk&agj9e|{;F-~8w`vcAP~ z*Xzp0VVVs-*CI|j2{JG<`A)lM;lk>l;rl4GLHSjUDfh@jEFNx z6jtbiZ<7`>@;7yCKUl@z1CJdlmykJn*1a{wi_AO12qn6@qlc|UW>oVqL1{r79BFFLK!-AhwokV}8jWZf%+ zce1{4w7O5&3m-3*Sgakikn@Ax?Hu6iJYJD@(mD~`d4keBLp{MUo%N`V04tlG^)x?%PZwOX>1a6=UXyy4VB%i@R%iA@#K_=N>QLX`m(OYQ| z)iK&sfufh%N8c7VOg9U2j#slcOyqs;TSRqGtNM$XtTJ_39cTdBuBET4lD)NQa?^r} z40Q}R!2CtV(H~l{>oRzc-=IEMVzk$=oVeofP=u2PXwcXTXa`+?0ks>s9~u%`naT4- z!WtBxh1Z2=pJb0q9fl*d^vt3&l;yQXdOsH~CY~)0G|teF(O1W!WJtC3K#W+UDLZ~B zXX>bE>cU9Mp%9<`{ON>3p|mnBz5zlFRz&;J={5cmRBp9(O24-|m5ht_9M5vL?^)JY zVfRQh8-*VW(1-2YCRXZOt8Tp=jc_d}9vMg(b4n7dFu&-eWaFit7CzohJk~I>VETKX z%UB`G0G7eutah0HWTKoCG?p7>nUnQBQc2wUwX{b$3S5VwQ&_-mq z$yc2i_>PZE@WE5@?3=iKvH!UX)EKX#ss`t&mK8b!QvFKgernYp{C}n|zfF&Ka2`FM zJ$(4rDYi*{BbR;t|OfRftOT9 za1FU?C_>2Sv=C->nZKRI8WQcx&out#`kbz&fqket2x7;VPly41*S1+q;RX@+<&ww_ zZP=YsZ|}1ou@BqNF|4fcSbku}6Ti{6x(H9E66U9*@Lc=f^W@&&hF)wGaiS@3D>;G*W!{>*GAgy)xTTqy=-is-K=$Zho0@;(W6PSx#?R|OTTfb!{V_>Kp=N;I5tWdFEUFUu~4MG}SkP6hIxj29FiWfYh$c|QsmfX)R^RkP~fUz#_M=uC<|N5Rg zb0iCOD5145wx-4Fc~6Ojq{!nl{w53S(D4gy80o-DAsuu;VY12NSO`Xt@z*8F1`T<_ zq(dq&(sl9L+RqQ>e<|r~+F3Zdde&LKf-uXJZ`uyn0=0GG^=A{tC7vmvs{yJUt<(;zi018^k0iW4JM4eeMb zYSI9SPTxNUT12ieU+w&(3vO~fWIJ1H#qmBjA(p;Eufhfwb^%B zp@saP1emjFajph3eQMv%sF5F|B4YM2mCMN7z*yW{zroF#KCj=)2+1bD<#tw7vYQ)9jk5_4r|Mdbu*l<#OCPP16id$Gb5sIG%T$G>QI}2x zu2pjFvnhsjG9Zgvs`M|q*(u+~yAAGGXGOImgLX^}yaHQpf@JbT<^UB`UGVFc02JGnW4xdqNx(q(?yUX?;7y|absgT`C7%0dUF`T*d~**L!a}FZiY%3HloWZ@ zeKSF(56TwV@Guy3YwR!6VeSYK?4ow$F%1voc%LP8)JP}^S7qI!>WJ7+ za+Iws93yz)AKc%e(W^?*K@96vH=Zf$_YSs0&soKPji9HqU^dLbB&`{jeoZXk4NlO) zo;9q?zW;OIbB3bzyXH!IlL$_@=q?uVe7U~hH_Hw9@JuN4O2|Vqucpv(L(2&%TmD

olsn%7+vELnH!%>9;n)5c=Bb2GiRr+5h5CP%>N0EHokP-l zw-5IqGNFvaRD5o>|N0>T+p>%0XY}fRS$ZR> Thiu@N7ijKj>fb5Ru#Nmb3T@G6 literal 0 HcmV?d00001 From 8385fec954612bade1cb947f7b72bbda37eb5652 Mon Sep 17 00:00:00 2001 From: Ravin Kumar Date: Tue, 18 Sep 2018 07:47:36 -0700 Subject: [PATCH 213/282] Fix small typo in docs (#2420) --- doc/related-projects.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/related-projects.rst b/doc/related-projects.rst index 714fbf98d7c..524ea3b9d8d 100644 --- a/doc/related-projects.rst +++ b/doc/related-projects.rst @@ -47,7 +47,7 @@ Extend xarray capabilities ~~~~~~~~~~~~~~~~~~~~~~~~~~ - `Collocate `_: Collocate xarray trajectories in arbitrary physical dimensions - `eofs `_: EOF analysis in Python. -- `xarray_extras `_: Advanced algorithms for xarray objects (e.g. intergrations/interpolations). +- `xarray_extras `_: Advanced algorithms for xarray objects (e.g. integrations/interpolations). - `xrft `_: Fourier transforms for xarray data. - `xr-scipy `_: A lightweight scipy wrapper for xarray. - `X-regression `_: Multiple linear regression from Statsmodels library coupled with Xarray library. From b679f4a9edb2437d11f28bf6516fc7aa8d673acb Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Tue, 18 Sep 2018 21:19:07 -0400 Subject: [PATCH 214/282] Adding data kwarg to copy to create new objects with same structure as original (#2384) * Added label_like for Variable and DataArray and tests * linting * Responding to comments - fixing up tests, more flexible input * label_like --> structured_like * Made docs changes, added example * Responding to comments * Moving from structured_like to .copy(data) * Making dataset copy mandate all data_vars and minor tweaks * Stop ignoring data in IndexVariable.copy --- doc/whats-new.rst | 4 ++ xarray/core/dataarray.py | 71 ++++++++++++++++++-- xarray/core/dataset.py | 114 +++++++++++++++++++++++++++++++-- xarray/core/variable.py | 114 ++++++++++++++++++++++++++++----- xarray/tests/test_dataarray.py | 12 ++++ xarray/tests/test_dataset.py | 21 ++++++ xarray/tests/test_variable.py | 28 ++++++++ 7 files changed, 338 insertions(+), 26 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 881ae52cdeb..b16533ad33b 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -76,6 +76,10 @@ Enhancements - You can now call ``unstack`` without arguments to unstack every MultiIndex in a DataArray or Dataset. By `Julia Signell `_. +- Added the ability to pass a data kwarg to ``copy`` to create a new object with the + same metadata as the original object but using new values. + By `Julia Signell `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index ae3758f0bbd..937d38d30fa 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -677,14 +677,77 @@ def persist(self, **kwargs): ds = self._to_temp_dataset().persist(**kwargs) return self._from_temp_dataset(ds) - def copy(self, deep=True): + def copy(self, deep=True, data=None): """Returns a copy of this array. - If `deep=True`, a deep copy is made of all variables in the underlying - dataset. Otherwise, a shallow copy is made, so each variable in the new + If `deep=True`, a deep copy is made of the data array. + Otherwise, a shallow copy is made, so each variable in the new array's dataset is also a variable in this array's dataset. + + Use `data` to create a new object with the same structure as + original but entirely new data. + + Parameters + ---------- + deep : bool, optional + Whether the data array and its coordinates are loaded into memory + and copied onto the new object. Default is True. + data : array_like, optional + Data to use in the new object. Must have same shape as original. + When `data` is used, `deep` is ignored for all data variables, + and only used for coords. + + Returns + ------- + object : DataArray + New object with dimensions, attributes, coordinates, name, + encoding, and optionally data copied from original. + + Examples + -------- + + Shallow versus deep copy + + >>> array = xr.DataArray([1, 2, 3], dims='x', + ... coords={'x': ['a', 'b', 'c']}) + >>> array.copy() + + array([1, 2, 3]) + Coordinates: + * x (x) >> array_0 = array.copy(deep=False) + >>> array_0[0] = 7 + >>> array_0 + + array([7, 2, 3]) + Coordinates: + * x (x) >> array + + array([7, 2, 3]) + Coordinates: + * x (x) >> array.copy(data=[0.1, 0.2, 0.3]) + + array([ 0.1, 0.2, 0.3]) + Coordinates: + * x (x) >> array + + array([1, 2, 3]) + Coordinates: + * x (x) >> da = xr.DataArray(np.random.randn(2, 3)) + >>> ds = xr.Dataset({'foo': da, 'bar': ('x', [-1, 2])}, + coords={'x': ['one', 'two']}) + >>> ds.copy() + + Dimensions: (dim_0: 2, dim_1: 3, x: 2) + Coordinates: + * x (x) >> ds_0 = ds.copy(deep=False) + >>> ds_0['foo'][0, 0] = 7 + >>> ds_0 + + Dimensions: (dim_0: 2, dim_1: 3, x: 2) + Coordinates: + * x (x) >> ds + + Dimensions: (dim_0: 2, dim_1: 3, x: 2) + Coordinates: + * x (x) >> ds.copy(data={'foo': np.arange(6).reshape(2, 3), 'bar': ['a', 'b']}) + + Dimensions: (dim_0: 2, dim_1: 3, x: 2) + Coordinates: + * x (x) >> ds + + Dimensions: (dim_0: 2, dim_1: 3, x: 2) + Coordinates: + * x (x) >> var = xr.Variable(data=[1, 2, 3], dims='x') + >>> var.copy() + + array([1, 2, 3]) + >>> var_0 = var.copy(deep=False) + >>> var_0[0] = 7 + >>> var_0 + + array([7, 2, 3]) + >>> var + + array([7, 2, 3]) + + Changing the data using the ``data`` argument maintains the + structure of the original object, but with the new data. Original + object is unaffected. + + >>> var.copy(data=[0.1, 0.2, 0.3]) + + array([ 0.1, 0.2, 0.3]) + >>> var + + array([7, 2, 3]) - if deep: - if isinstance(data, dask_array_type): - data = data.copy() - elif not isinstance(data, PandasIndexAdapter): - # pandas.Index is immutable - data = np.array(data) + See Also + -------- + pandas.DataFrame.copy + """ + if data is None: + data = self._data + + if isinstance(data, indexing.MemoryCachedArray): + # don't share caching between copies + data = indexing.MemoryCachedArray(data.array) + + if deep: + if isinstance(data, dask_array_type): + data = data.copy() + elif not isinstance(data, PandasIndexAdapter): + # pandas.Index is immutable + data = np.array(data) + else: + data = as_compatible_data(data) + if self.shape != data.shape: + raise ValueError("Data shape {} must match shape of object {}" + .format(data.shape, self.shape)) # note: # dims is already an immutable tuple @@ -1709,14 +1766,37 @@ def concat(cls, variables, dim='concat_dim', positions=None, return cls(first_var.dims, data, attrs) - def copy(self, deep=True): + def copy(self, deep=True, data=None): """Returns a copy of this object. - `deep` is ignored since data is stored in the form of pandas.Index, - which is already immutable. Dimensions, attributes and encodings are - always copied. + `deep` is ignored since data is stored in the form of + pandas.Index, which is already immutable. Dimensions, attributes + and encodings are always copied. + + Use `data` to create a new object with the same structure as + original but entirely new data. + + Parameters + ---------- + deep : bool, optional + Deep is always ignored. + data : array_like, optional + Data to use in the new object. Must have same shape as original. + + Returns + ------- + object : Variable + New object with dimensions, attributes, encodings, and optionally + data copied from original. """ - return type(self)(self.dims, self._data, self._attrs, + if data is None: + data = self._data + else: + data = as_compatible_data(data) + if self.shape != data.shape: + raise ValueError("Data shape {} must match shape of object {}" + .format(data.shape, self.shape)) + return type(self)(self.dims, data, self._attrs, self._encoding, fastpath=True) def equals(self, other, equiv=None): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index a4562894583..2b93e696d50 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3138,6 +3138,18 @@ def test_roll_coords_none(self): expected = DataArray([3, 1, 2], coords=[('x', [2, 0, 1])]) assert_identical(expected, actual) + def test_copy_with_data(self): + orig = DataArray(np.random.random(size=(2, 2)), + dims=('x', 'y'), + attrs={'attr1': 'value1'}, + coords={'x': [4, 3]}, + name='helloworld') + new_data = np.arange(4).reshape(2, 2) + actual = orig.copy(data=new_data) + expected = orig.copy() + expected.data = new_data + assert_identical(expected, actual) + def test_real_and_imag(self): array = DataArray(1 + 2j) assert_identical(array.real, DataArray(1)) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index d22d8470dc6..fc933960914 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1892,6 +1892,27 @@ def test_copy(self): v1 = copied.variables[k] assert v0 is not v1 + def test_copy_with_data(self): + orig = create_test_data() + new_data = {k: np.random.randn(*v.shape) + for k, v in iteritems(orig.data_vars)} + actual = orig.copy(data=new_data) + + expected = orig.copy() + for k, v in new_data.items(): + expected[k].data = v + assert_identical(expected, actual) + + def test_copy_with_data_errors(self): + orig = create_test_data() + new_var1 = np.arange(orig['var1'].size).reshape(orig['var1'].shape) + with raises_regex(ValueError, 'Data must be dict-like'): + orig.copy(data=new_var1) + with raises_regex(ValueError, 'only contain variables in original'): + orig.copy(data={'not_in_original': new_var1}) + with raises_regex(ValueError, 'contain all variables in original'): + orig.copy(data={'var1': new_var1}) + def test_rename(self): data = create_test_data() newnames = {'var1': 'renamed_var1', 'dim2': 'renamed_dim2'} diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 904940cbbf6..1263ac1df9e 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -505,6 +505,34 @@ def test_copy_index(self): assert isinstance(w.to_index(), pd.MultiIndex) assert_array_equal(v._data.array, w._data.array) + def test_copy_with_data(self): + orig = Variable(('x', 'y'), [[1.5, 2.0], [3.1, 4.3]], {'foo': 'bar'}) + new_data = np.array([[2.5, 5.0], [7.1, 43]]) + actual = orig.copy(data=new_data) + expected = orig.copy() + expected.data = new_data + assert_identical(expected, actual) + + def test_copy_with_data_errors(self): + orig = Variable(('x', 'y'), [[1.5, 2.0], [3.1, 4.3]], {'foo': 'bar'}) + new_data = [2.5, 5.0] + with raises_regex(ValueError, 'must match shape of object'): + orig.copy(data=new_data) + + def test_copy_index_with_data(self): + orig = IndexVariable('x', np.arange(5)) + new_data = np.arange(5, 10) + actual = orig.copy(data=new_data) + expected = orig.copy() + expected.data = new_data + assert_identical(expected, actual) + + def test_copy_index_with_data_errors(self): + orig = IndexVariable('x', np.arange(5)) + new_data = np.arange(5, 20) + with raises_regex(ValueError, 'must match shape of object'): + orig.copy(data=new_data) + def test_real_and_imag(self): v = self.cls('x', np.arange(3) - 1j * np.arange(3), {'foo': 'bar'}) expected_re = self.cls('x', np.arange(3), {'foo': 'bar'}) From a0b5af5a1945ccac3704df0ff2acaf55f2db2de6 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 18 Sep 2018 22:59:27 -0700 Subject: [PATCH 215/282] Update NumFOCUS donate link (#2421) * add some blurbs about numfocus sponsorship to docs * add numfocus to history blurb * add missing logo file * markdown to rst in readme * update numfocus donate links in documentation * more links to numfocus --- README.rst | 5 +++-- doc/index.rst | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 12650b1db1b..0ac71d33954 100644 --- a/README.rst +++ b/README.rst @@ -110,13 +110,14 @@ NumFOCUS .. image:: https://numfocus.org/wp-content/uploads/2017/07/NumFocus_LRG.png :scale: 25 % + :target: https://numfocus.org/ -Xarray is a fiscally sponsored project of NumFOCUS, a nonprofit dedicated +Xarray is a fiscally sponsored project of NumFOCUS_, a nonprofit dedicated to supporting the open source scientific computing community. If you like Xarray and want to support our mission, please consider making a donation_ to support our efforts. -.. _donation: https://www.flipcause.com/secure/cause_pdetails/MjE3OQ== +.. _donation: https://www.flipcause.com/secure/cause_pdetails/NDE2NTU= History ------- diff --git a/doc/index.rst b/doc/index.rst index 6c1a8519507..45897f4bccb 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -125,13 +125,16 @@ NumFOCUS .. image:: _static/numfocus_logo.png :scale: 50 % + :target: https://numfocus.org/ -Xarray is a fiscally sponsored project of NumFOCUS, a nonprofit dedicated +Xarray is a fiscally sponsored project of NumFOCUS_, a nonprofit dedicated to supporting the open source scientific computing community. If you like -Xarray and want to support our mission, please consider making a -[donation](https://www.flipcause.com/secure/cause_pdetails/MjE3OQ==) +Xarray and want to support our mission, please consider making a donation_ to support our efforts. +.. _donation: https://www.flipcause.com/secure/cause_pdetails/NDE2NTU= + + History ------- From 5b87b6e2f159b827f739e12d4faae57a0b6f6178 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Wed, 19 Sep 2018 16:24:39 -0400 Subject: [PATCH 216/282] WIP Add a CFTimeIndex-enabled xr.cftime_range function (#2301) * Initial work on adding a cftime-compatible date_range function Add docstring for xr.date_range Fix failing test Fix test skipping logic Coerce result of zip to a list in test setup Add and clean up tests Fix skip logic Skip roll_forward and roll_backward tests if cftime is not installed Expose all possible arguments to pd.date_range Add more detail to docstrings flake8 Add a what's new entry Add a short example to time-series.rst * Allow empty CFTimeIndexes; add calendar to CFTimeIndex repr * Enable CFTimeIndex constructor to optionally take date_range arguments * Simplify date_range to use new CFTimeIndex constructor * Rename xr.date_range to xr.cftime_range * Follow pandas behavior/naming for rollforward, rollback, and onOffset * Update docstring * Add pandas copyright notice to cftime_offsets.py * Check validity of offset constructor arguments * Fix TypeError versus ValueError uses * Use a module-level importorskip in test_cftime_offsets.py * Only return a CFTimeIndex from cftime_range * Keep CFTimeIndex constructor simple * Add some explicitly calendar-specific tests * Revert back to default repr * lint * return NotImplemented * Convert pandas copyright notices to comments * test_calendar_leap_year_length -> test_calendar_year_length * Use return NotImplemented in __apply__ too --- doc/api.rst | 7 + doc/time-series.rst | 11 +- doc/whats-new.rst | 3 + xarray/__init__.py | 1 + xarray/coding/cftime_offsets.py | 736 +++++++++++++++++++++++++ xarray/coding/cftimeindex.py | 88 ++- xarray/tests/test_cftime_offsets.py | 801 ++++++++++++++++++++++++++++ xarray/tests/test_cftimeindex.py | 39 +- 8 files changed, 1665 insertions(+), 21 deletions(-) create mode 100644 xarray/coding/cftime_offsets.py create mode 100644 xarray/tests/test_cftime_offsets.py diff --git a/doc/api.rst b/doc/api.rst index 927c0aa072c..89fee10506d 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -555,6 +555,13 @@ Custom Indexes CFTimeIndex +Creating custom indexes +----------------------- +.. autosummary:: + :toctree: generated/ + + cftime_range + Plotting ======== diff --git a/doc/time-series.rst b/doc/time-series.rst index a7ce9226d4d..d99c3218d18 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -258,7 +258,16 @@ coordinate with a no-leap calendar within a context manager setting the calendar, its times will be decoded into ``cftime.datetime`` objects, regardless of whether or not they can be represented using ``np.datetime64[ns]`` objects. - + +xarray also includes a :py:func:`cftime_range` function, which enables creating a +``CFTimeIndex`` with regularly-spaced dates. For instance, we can create the +same dates and DataArray we created above using: + +.. ipython:: python + + dates = xr.cftime_range(start='0001', periods=24, freq='MS', calendar='noleap') + da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], name='foo') + For data indexed by a ``CFTimeIndex`` xarray currently supports: - `Partial datetime string indexing`_ using strictly `ISO 8601-format`_ partial diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b16533ad33b..8c34ddf3fa9 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -54,6 +54,9 @@ Enhancements now displayed as `a b ... y z` rather than `a b c d ...`. (:issue:`1186`) By `Seth P `_. +- A new CFTimeIndex-enabled :py:func:`cftime_range` function for use in + generating dates from standard or non-standard calendars. By `Spencer Clark + `_. - When interpolating over a ``datetime64`` axis, you can now provide a datetime string instead of a ``datetime64`` object. E.g. ``da.interp(time='1991-02-01')`` (:issue:`2284`) diff --git a/xarray/__init__.py b/xarray/__init__.py index 7cc7811b783..e2d24e6c294 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -26,6 +26,7 @@ from .conventions import decode_cf, SerializationWarning +from .coding.cftime_offsets import cftime_range from .coding.cftimeindex import CFTimeIndex from .util.print_versions import show_versions diff --git a/xarray/coding/cftime_offsets.py b/xarray/coding/cftime_offsets.py new file mode 100644 index 00000000000..3fbb44f4ed3 --- /dev/null +++ b/xarray/coding/cftime_offsets.py @@ -0,0 +1,736 @@ +"""Time offset classes for use with cftime.datetime objects""" +# The offset classes and mechanisms for generating time ranges defined in +# this module were copied/adapted from those defined in pandas. See in +# particular the objects and methods defined in pandas.tseries.offsets +# and pandas.core.indexes.datetimes. + +# For reference, here is a copy of the pandas copyright notice: + +# (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +# All rights reserved. + +# Copyright (c) 2008-2011 AQR Capital Management, LLC +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. + +# * Neither the name of the copyright holder nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import re + +from datetime import timedelta +from functools import partial + +import numpy as np + +from .cftimeindex import _parse_iso8601_with_reso, CFTimeIndex +from .times import format_cftime_datetime +from ..core.pycompat import basestring + + +def get_date_type(calendar): + """Return the cftime date type for a given calendar name.""" + try: + import cftime + except ImportError: + raise ImportError( + 'cftime is required for dates with non-standard calendars') + else: + calendars = { + 'noleap': cftime.DatetimeNoLeap, + '360_day': cftime.Datetime360Day, + '365_day': cftime.DatetimeNoLeap, + '366_day': cftime.DatetimeAllLeap, + 'gregorian': cftime.DatetimeGregorian, + 'proleptic_gregorian': cftime.DatetimeProlepticGregorian, + 'julian': cftime.DatetimeJulian, + 'all_leap': cftime.DatetimeAllLeap, + 'standard': cftime.DatetimeProlepticGregorian + } + return calendars[calendar] + + +class BaseCFTimeOffset(object): + _freq = None + + def __init__(self, n=1): + if not isinstance(n, int): + raise TypeError( + "The provided multiple 'n' must be an integer. " + "Instead a value of type {!r} was provided.".format(type(n))) + self.n = n + + def rule_code(self): + return self._freq + + def __eq__(self, other): + return self.n == other.n and self.rule_code() == other.rule_code() + + def __ne__(self, other): + return not self == other + + def __add__(self, other): + return self.__apply__(other) + + def __sub__(self, other): + import cftime + + if isinstance(other, cftime.datetime): + raise TypeError('Cannot subtract a cftime.datetime ' + 'from a time offset.') + elif type(other) == type(self): + return type(self)(self.n - other.n) + else: + return NotImplemented + + def __mul__(self, other): + return type(self)(n=other * self.n) + + def __neg__(self): + return self * -1 + + def __rmul__(self, other): + return self.__mul__(other) + + def __radd__(self, other): + return self.__add__(other) + + def __rsub__(self, other): + if isinstance(other, BaseCFTimeOffset) and type(self) != type(other): + raise TypeError('Cannot subtract cftime offsets of differing ' + 'types') + return -self + other + + def __apply__(self): + return NotImplemented + + def onOffset(self, date): + """Check if the given date is in the set of possible dates created + using a length-one version of this offset class.""" + test_date = (self + date) - self + return date == test_date + + def rollforward(self, date): + if self.onOffset(date): + return date + else: + return date + type(self)() + + def rollback(self, date): + if self.onOffset(date): + return date + else: + return date - type(self)() + + def __str__(self): + return '<{}: n={}>'.format(type(self).__name__, self.n) + + def __repr__(self): + return str(self) + + +def _days_in_month(date): + """The number of days in the month of the given date""" + if date.month == 12: + reference = type(date)(date.year + 1, 1, 1) + else: + reference = type(date)(date.year, date.month + 1, 1) + return (reference - timedelta(days=1)).day + + +def _adjust_n_months(other_day, n, reference_day): + """Adjust the number of times a monthly offset is applied based + on the day of a given date, and the reference day provided. + """ + if n > 0 and other_day < reference_day: + n = n - 1 + elif n <= 0 and other_day > reference_day: + n = n + 1 + return n + + +def _adjust_n_years(other, n, month, reference_day): + """Adjust the number of times an annual offset is applied based on + another date, and the reference day provided""" + if n > 0: + if other.month < month or (other.month == month and + other.day < reference_day): + n -= 1 + else: + if other.month > month or (other.month == month and + other.day > reference_day): + n += 1 + return n + + +def _shift_months(date, months, day_option='start'): + """Shift the date to a month start or end a given number of months away. + """ + delta_year = (date.month + months) // 12 + month = (date.month + months) % 12 + + if month == 0: + month = 12 + delta_year = delta_year - 1 + year = date.year + delta_year + + if day_option == 'start': + day = 1 + elif day_option == 'end': + reference = type(date)(year, month, 1) + day = _days_in_month(reference) + else: + raise ValueError(day_option) + return date.replace(year=year, month=month, day=day) + + +class MonthBegin(BaseCFTimeOffset): + _freq = 'MS' + + def __apply__(self, other): + n = _adjust_n_months(other.day, self.n, 1) + return _shift_months(other, n, 'start') + + def onOffset(self, date): + """Check if the given date is in the set of possible dates created + using a length-one version of this offset class.""" + return date.day == 1 + + +class MonthEnd(BaseCFTimeOffset): + _freq = 'M' + + def __apply__(self, other): + n = _adjust_n_months(other.day, self.n, _days_in_month(other)) + return _shift_months(other, n, 'end') + + def onOffset(self, date): + """Check if the given date is in the set of possible dates created + using a length-one version of this offset class.""" + return date.day == _days_in_month(date) + + +_MONTH_ABBREVIATIONS = { + 1: 'JAN', + 2: 'FEB', + 3: 'MAR', + 4: 'APR', + 5: 'MAY', + 6: 'JUN', + 7: 'JUL', + 8: 'AUG', + 9: 'SEP', + 10: 'OCT', + 11: 'NOV', + 12: 'DEC' +} + + +class YearOffset(BaseCFTimeOffset): + _freq = None + _day_option = None + _default_month = None + + def __init__(self, n=1, month=None): + BaseCFTimeOffset.__init__(self, n) + if month is None: + self.month = self._default_month + else: + self.month = month + if not isinstance(self.month, int): + raise TypeError("'self.month' must be an integer value between 1 " + "and 12. Instead, it was set to a value of " + "{!r}".format(self.month)) + elif not (1 <= self.month <= 12): + raise ValueError("'self.month' must be an integer value between 1 " + "and 12. Instead, it was set to a value of " + "{!r}".format(self.month)) + + def __apply__(self, other): + if self._day_option == 'start': + reference_day = 1 + elif self._day_option == 'end': + reference_day = _days_in_month(other) + else: + raise ValueError(self._day_option) + years = _adjust_n_years(other, self.n, self.month, reference_day) + months = years * 12 + (self.month - other.month) + return _shift_months(other, months, self._day_option) + + def __sub__(self, other): + import cftime + + if isinstance(other, cftime.datetime): + raise TypeError('Cannot subtract cftime.datetime from offset.') + elif type(other) == type(self) and other.month == self.month: + return type(self)(self.n - other.n, month=self.month) + else: + return NotImplemented + + def __mul__(self, other): + return type(self)(n=other * self.n, month=self.month) + + def rule_code(self): + return '{}-{}'.format(self._freq, _MONTH_ABBREVIATIONS[self.month]) + + def __str__(self): + return '<{}: n={}, month={}>'.format( + type(self).__name__, self.n, self.month) + + +class YearBegin(YearOffset): + _freq = 'AS' + _day_option = 'start' + _default_month = 1 + + def onOffset(self, date): + """Check if the given date is in the set of possible dates created + using a length-one version of this offset class.""" + return date.day == 1 and date.month == self.month + + def rollforward(self, date): + """Roll date forward to nearest start of year""" + if self.onOffset(date): + return date + else: + return date + YearBegin(month=self.month) + + def rollback(self, date): + """Roll date backward to nearest start of year""" + if self.onOffset(date): + return date + else: + return date - YearBegin(month=self.month) + + +class YearEnd(YearOffset): + _freq = 'A' + _day_option = 'end' + _default_month = 12 + + def onOffset(self, date): + """Check if the given date is in the set of possible dates created + using a length-one version of this offset class.""" + return date.day == _days_in_month(date) and date.month == self.month + + def rollforward(self, date): + """Roll date forward to nearest end of year""" + if self.onOffset(date): + return date + else: + return date + YearEnd(month=self.month) + + def rollback(self, date): + """Roll date backward to nearest end of year""" + if self.onOffset(date): + return date + else: + return date - YearEnd(month=self.month) + + +class Day(BaseCFTimeOffset): + _freq = 'D' + + def __apply__(self, other): + return other + timedelta(days=self.n) + + +class Hour(BaseCFTimeOffset): + _freq = 'H' + + def __apply__(self, other): + return other + timedelta(hours=self.n) + + +class Minute(BaseCFTimeOffset): + _freq = 'T' + + def __apply__(self, other): + return other + timedelta(minutes=self.n) + + +class Second(BaseCFTimeOffset): + _freq = 'S' + + def __apply__(self, other): + return other + timedelta(seconds=self.n) + + +_FREQUENCIES = { + 'A': YearEnd, + 'AS': YearBegin, + 'Y': YearEnd, + 'YS': YearBegin, + 'M': MonthEnd, + 'MS': MonthBegin, + 'D': Day, + 'H': Hour, + 'T': Minute, + 'min': Minute, + 'S': Second, + 'AS-JAN': partial(YearBegin, month=1), + 'AS-FEB': partial(YearBegin, month=2), + 'AS-MAR': partial(YearBegin, month=3), + 'AS-APR': partial(YearBegin, month=4), + 'AS-MAY': partial(YearBegin, month=5), + 'AS-JUN': partial(YearBegin, month=6), + 'AS-JUL': partial(YearBegin, month=7), + 'AS-AUG': partial(YearBegin, month=8), + 'AS-SEP': partial(YearBegin, month=9), + 'AS-OCT': partial(YearBegin, month=10), + 'AS-NOV': partial(YearBegin, month=11), + 'AS-DEC': partial(YearBegin, month=12), + 'A-JAN': partial(YearEnd, month=1), + 'A-FEB': partial(YearEnd, month=2), + 'A-MAR': partial(YearEnd, month=3), + 'A-APR': partial(YearEnd, month=4), + 'A-MAY': partial(YearEnd, month=5), + 'A-JUN': partial(YearEnd, month=6), + 'A-JUL': partial(YearEnd, month=7), + 'A-AUG': partial(YearEnd, month=8), + 'A-SEP': partial(YearEnd, month=9), + 'A-OCT': partial(YearEnd, month=10), + 'A-NOV': partial(YearEnd, month=11), + 'A-DEC': partial(YearEnd, month=12) +} + + +_FREQUENCY_CONDITION = '|'.join(_FREQUENCIES.keys()) +_PATTERN = '^((?P\d+)|())(?P({0}))$'.format( + _FREQUENCY_CONDITION) + + +def to_offset(freq): + """Convert a frequency string to the appropriate subclass of + BaseCFTimeOffset.""" + if isinstance(freq, BaseCFTimeOffset): + return freq + else: + try: + freq_data = re.match(_PATTERN, freq).groupdict() + except AttributeError: + raise ValueError('Invalid frequency string provided') + + freq = freq_data['freq'] + multiples = freq_data['multiple'] + if multiples is None: + multiples = 1 + else: + multiples = int(multiples) + + return _FREQUENCIES[freq](n=multiples) + + +def to_cftime_datetime(date_str_or_date, calendar=None): + import cftime + + if isinstance(date_str_or_date, basestring): + if calendar is None: + raise ValueError( + 'If converting a string to a cftime.datetime object, ' + 'a calendar type must be provided') + date, _ = _parse_iso8601_with_reso(get_date_type(calendar), + date_str_or_date) + return date + elif isinstance(date_str_or_date, cftime.datetime): + return date_str_or_date + else: + raise TypeError("date_str_or_date must be a string or a " + 'subclass of cftime.datetime. Instead got ' + '{!r}.'.format(date_str_or_date)) + + +def normalize_date(date): + """Round datetime down to midnight.""" + return date.replace(hour=0, minute=0, second=0, microsecond=0) + + +def _maybe_normalize_date(date, normalize): + """Round datetime down to midnight if normalize is True.""" + if normalize: + return normalize_date(date) + else: + return date + + +def _generate_linear_range(start, end, periods): + """Generate an equally-spaced sequence of cftime.datetime objects between + and including two dates (whose length equals the number of periods).""" + import cftime + + total_seconds = (end - start).total_seconds() + values = np.linspace(0., total_seconds, periods, endpoint=True) + units = 'seconds since {}'.format(format_cftime_datetime(start)) + calendar = start.calendar + return cftime.num2date(values, units=units, calendar=calendar, + only_use_cftime_datetimes=True) + + +def _generate_range(start, end, periods, offset): + """Generate a regular range of cftime.datetime objects with a + given time offset. + + Adapted from pandas.tseries.offsets.generate_range. + + Parameters + ---------- + start : cftime.datetime, or None + Start of range + end : cftime.datetime, or None + End of range + periods : int, or None + Number of elements in the sequence + offset : BaseCFTimeOffset + An offset class designed for working with cftime.datetime objects + + Returns + ------- + A generator object + """ + if start: + start = offset.rollforward(start) + + if end: + end = offset.rollback(end) + + if periods is None and end < start: + end = None + periods = 0 + + if end is None: + end = start + (periods - 1) * offset + + if start is None: + start = end - (periods - 1) * offset + + current = start + if offset.n >= 0: + while current <= end: + yield current + + next_date = current + offset + if next_date <= current: + raise ValueError('Offset {offset} did not increment date' + .format(offset=offset)) + current = next_date + else: + while current >= end: + yield current + + next_date = current + offset + if next_date >= current: + raise ValueError('Offset {offset} did not decrement date' + .format(offset=offset)) + current = next_date + + +def _count_not_none(*args): + """Compute the number of non-None arguments.""" + return sum([arg is not None for arg in args]) + + +def cftime_range(start=None, end=None, periods=None, freq='D', + tz=None, normalize=False, name=None, closed=None, + calendar='standard'): + """Return a fixed frequency CFTimeIndex. + + Parameters + ---------- + start : str or cftime.datetime, optional + Left bound for generating dates. + end : str or cftime.datetime, optional + Right bound for generating dates. + periods : integer, optional + Number of periods to generate. + freq : str, default 'D', BaseCFTimeOffset, or None + Frequency strings can have multiples, e.g. '5H'. + normalize : bool, default False + Normalize start/end dates to midnight before generating date range. + name : str, default None + Name of the resulting index + closed : {None, 'left', 'right'}, optional + Make the interval closed with respect to the given frequency to the + 'left', 'right', or both sides (None, the default). + calendar : str + Calendar type for the datetimes (default 'standard'). + + Returns + ------- + CFTimeIndex + + Notes + ----- + + This function is an analog of ``pandas.date_range`` for use in generating + sequences of ``cftime.datetime`` objects. It supports most of the + features of ``pandas.date_range`` (e.g. specifying how the index is + ``closed`` on either side, or whether or not to ``normalize`` the start and + end bounds); however, there are some notable exceptions: + + - You cannot specify a ``tz`` (time zone) argument. + - Start or end dates specified as partial-datetime strings must use the + `ISO-8601 format `_. + - It supports many, but not all, frequencies supported by + ``pandas.date_range``. For example it does not currently support any of + the business-related, semi-monthly, or sub-second frequencies. + - Compound sub-monthly frequencies are not supported, e.g. '1H1min', as + these can easily be written in terms of the finest common resolution, + e.g. '61min'. + + Valid simple frequency strings for use with ``cftime``-calendars include + any multiples of the following. + + +--------+-----------------------+ + | Alias | Description | + +========+=======================+ + | A, Y | Year-end frequency | + +--------+-----------------------+ + | AS, YS | Year-start frequency | + +--------+-----------------------+ + | M | Month-end frequency | + +--------+-----------------------+ + | MS | Month-start frequency | + +--------+-----------------------+ + | D | Day frequency | + +--------+-----------------------+ + | H | Hour frequency | + +--------+-----------------------+ + | T, min | Minute frequency | + +--------+-----------------------+ + | S | Second frequency | + +--------+-----------------------+ + + Any multiples of the following anchored offsets are also supported. + + +----------+-------------------------------------------------------------------+ + | Alias | Description | + +==========+===================================================================+ + | A(S)-JAN | Annual frequency, anchored at the end (or beginning) of January | + +----------+-------------------------------------------------------------------+ + | A(S)-FEB | Annual frequency, anchored at the end (or beginning) of February | + +----------+-------------------------------------------------------------------+ + | A(S)-MAR | Annual frequency, anchored at the end (or beginning) of March | + +----------+-------------------------------------------------------------------+ + | A(S)-APR | Annual frequency, anchored at the end (or beginning) of April | + +----------+-------------------------------------------------------------------+ + | A(S)-MAY | Annual frequency, anchored at the end (or beginning) of May | + +----------+-------------------------------------------------------------------+ + | A(S)-JUN | Annual frequency, anchored at the end (or beginning) of June | + +----------+-------------------------------------------------------------------+ + | A(S)-JUL | Annual frequency, anchored at the end (or beginning) of July | + +----------+-------------------------------------------------------------------+ + | A(S)-AUG | Annual frequency, anchored at the end (or beginning) of August | + +----------+-------------------------------------------------------------------+ + | A(S)-SEP | Annual frequency, anchored at the end (or beginning) of September | + +----------+-------------------------------------------------------------------+ + | A(S)-OCT | Annual frequency, anchored at the end (or beginning) of October | + +----------+-------------------------------------------------------------------+ + | A(S)-NOV | Annual frequency, anchored at the end (or beginning) of November | + +----------+-------------------------------------------------------------------+ + | A(S)-DEC | Annual frequency, anchored at the end (or beginning) of December | + +----------+-------------------------------------------------------------------+ + + Finally, the following calendar aliases are supported. + + +--------------------------------+---------------------------------------+ + | Alias | Date type | + +================================+=======================================+ + | standard, proleptic_gregorian | ``cftime.DatetimeProlepticGregorian`` | + +--------------------------------+---------------------------------------+ + | gregorian | ``cftime.DatetimeGregorian`` | + +--------------------------------+---------------------------------------+ + | noleap, 365_day | ``cftime.DatetimeNoLeap`` | + +--------------------------------+---------------------------------------+ + | all_leap, 366_day | ``cftime.DatetimeAllLeap`` | + +--------------------------------+---------------------------------------+ + | 360_day | ``cftime.Datetime360Day`` | + +--------------------------------+---------------------------------------+ + | julian | ``cftime.DatetimeJulian`` | + +--------------------------------+---------------------------------------+ + + Examples + -------- + + This function returns a ``CFTimeIndex``, populated with ``cftime.datetime`` + objects associated with the specified calendar type, e.g. + + >>> xr.cftime_range(start='2000', periods=6, freq='2MS', calendar='noleap') + CFTimeIndex([2000-01-01 00:00:00, 2000-03-01 00:00:00, 2000-05-01 00:00:00, + 2000-07-01 00:00:00, 2000-09-01 00:00:00, 2000-11-01 00:00:00], + dtype='object') + + As in the standard pandas function, three of the ``start``, ``end``, + ``periods``, or ``freq`` arguments must be specified at a given time, with + the other set to ``None``. See the `pandas documentation + `_ + for more examples of the behavior of ``date_range`` with each of the + parameters. + + See Also + -------- + pandas.date_range + """ # noqa: E501 + # Adapted from pandas.core.indexes.datetimes._generate_range. + if _count_not_none(start, end, periods, freq) != 3: + raise ValueError( + "Of the arguments 'start', 'end', 'periods', and 'freq', three " + "must be specified at a time.") + + if start is not None: + start = to_cftime_datetime(start, calendar) + start = _maybe_normalize_date(start, normalize) + if end is not None: + end = to_cftime_datetime(end, calendar) + end = _maybe_normalize_date(end, normalize) + + if freq is None: + dates = _generate_linear_range(start, end, periods) + else: + offset = to_offset(freq) + dates = np.array(list(_generate_range(start, end, periods, offset))) + + left_closed = False + right_closed = False + + if closed is None: + left_closed = True + right_closed = True + elif closed == 'left': + left_closed = True + elif closed == 'right': + right_closed = True + else: + raise ValueError("Closed must be either 'left', 'right' or None") + + if (not left_closed and len(dates) and + start is not None and dates[0] == start): + dates = dates[1:] + if (not right_closed and len(dates) and + end is not None and dates[-1] == end): + dates = dates[:-1] + + return CFTimeIndex(dates, name=name) diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index eb8cae2f398..ea2bcbc5858 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -1,3 +1,44 @@ +"""DatetimeIndex analog for cftime.datetime objects""" +# The pandas.Index subclass defined here was copied and adapted for +# use with cftime.datetime objects based on the source code defining +# pandas.DatetimeIndex. + +# For reference, here is a copy of the pandas copyright notice: + +# (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +# All rights reserved. + +# Copyright (c) 2008-2011 AQR Capital Management, LLC +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. + +# * Neither the name of the copyright holder nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import absolute_import import re from datetime import timedelta @@ -116,28 +157,43 @@ def f(self): def get_date_type(self): - return type(self._data[0]) + if self.data: + return type(self._data[0]) + else: + return None def assert_all_valid_date_type(data): import cftime - sample = data[0] - date_type = type(sample) - if not isinstance(sample, cftime.datetime): - raise TypeError( - 'CFTimeIndex requires cftime.datetime ' - 'objects. Got object of {}.'.format(date_type)) - if not all(isinstance(value, date_type) for value in data): - raise TypeError( - 'CFTimeIndex requires using datetime ' - 'objects of all the same type. Got\n{}.'.format(data)) + if data.size: + sample = data[0] + date_type = type(sample) + if not isinstance(sample, cftime.datetime): + raise TypeError( + 'CFTimeIndex requires cftime.datetime ' + 'objects. Got object of {}.'.format(date_type)) + if not all(isinstance(value, date_type) for value in data): + raise TypeError( + 'CFTimeIndex requires using datetime ' + 'objects of all the same type. Got\n{}.'.format(data)) class CFTimeIndex(pd.Index): """Custom Index for working with CF calendars and dates All elements of a CFTimeIndex must be cftime.datetime objects. + + Parameters + ---------- + data : array or CFTimeIndex + Sequence of cftime.datetime objects to use in index + name : str, default None + Name of the resulting index + + See Also + -------- + cftime_range """ year = _field_accessor('year', 'The year of the datetime') month = _field_accessor('month', 'The month of the datetime') @@ -149,10 +205,14 @@ class CFTimeIndex(pd.Index): 'The microseconds of the datetime') date_type = property(get_date_type) - def __new__(cls, data): + def __new__(cls, data, name=None): + if name is None and hasattr(data, 'name'): + name = data.name + result = object.__new__(cls) - assert_all_valid_date_type(data) - result._data = np.array(data) + result._data = np.array(data, dtype='O') + assert_all_valid_date_type(result._data) + result.name = name return result def _partial_date_slice(self, resolution, parsed): diff --git a/xarray/tests/test_cftime_offsets.py b/xarray/tests/test_cftime_offsets.py new file mode 100644 index 00000000000..6d7990689ed --- /dev/null +++ b/xarray/tests/test_cftime_offsets.py @@ -0,0 +1,801 @@ +import pytest + +from itertools import product + +import numpy as np + +from xarray.coding.cftime_offsets import ( + BaseCFTimeOffset, YearBegin, YearEnd, MonthBegin, MonthEnd, + Day, Hour, Minute, Second, _days_in_month, + to_offset, get_date_type, _MONTH_ABBREVIATIONS, to_cftime_datetime, + cftime_range) +from xarray import CFTimeIndex + +cftime = pytest.importorskip('cftime') + + +_CFTIME_CALENDARS = ['365_day', '360_day', 'julian', 'all_leap', + '366_day', 'gregorian', 'proleptic_gregorian', 'standard'] + + +def _id_func(param): + """Called on each parameter passed to pytest.mark.parametrize""" + return str(param) + + +@pytest.fixture(params=_CFTIME_CALENDARS) +def calendar(request): + return request.param + + +@pytest.mark.parametrize( + ('offset', 'expected_n'), + [(BaseCFTimeOffset(), 1), + (YearBegin(), 1), + (YearEnd(), 1), + (BaseCFTimeOffset(n=2), 2), + (YearBegin(n=2), 2), + (YearEnd(n=2), 2)], + ids=_id_func +) +def test_cftime_offset_constructor_valid_n(offset, expected_n): + assert offset.n == expected_n + + +@pytest.mark.parametrize( + ('offset', 'invalid_n'), + [(BaseCFTimeOffset, 1.5), + (YearBegin, 1.5), + (YearEnd, 1.5)], + ids=_id_func +) +def test_cftime_offset_constructor_invalid_n(offset, invalid_n): + with pytest.raises(TypeError): + offset(n=invalid_n) + + +@pytest.mark.parametrize( + ('offset', 'expected_month'), + [(YearBegin(), 1), + (YearEnd(), 12), + (YearBegin(month=5), 5), + (YearEnd(month=5), 5)], + ids=_id_func +) +def test_year_offset_constructor_valid_month(offset, expected_month): + assert offset.month == expected_month + + +@pytest.mark.parametrize( + ('offset', 'invalid_month', 'exception'), + [(YearBegin, 0, ValueError), + (YearEnd, 0, ValueError), + (YearBegin, 13, ValueError,), + (YearEnd, 13, ValueError), + (YearBegin, 1.5, TypeError), + (YearEnd, 1.5, TypeError)], + ids=_id_func +) +def test_year_offset_constructor_invalid_month( + offset, invalid_month, exception): + with pytest.raises(exception): + offset(month=invalid_month) + + +@pytest.mark.parametrize( + ('offset', 'expected'), + [(BaseCFTimeOffset(), None), + (MonthBegin(), 'MS'), + (YearBegin(), 'AS-JAN')], + ids=_id_func +) +def test_rule_code(offset, expected): + assert offset.rule_code() == expected + + +@pytest.mark.parametrize( + ('offset', 'expected'), + [(BaseCFTimeOffset(), ''), + (YearBegin(), '')], + ids=_id_func +) +def test_str_and_repr(offset, expected): + assert str(offset) == expected + assert repr(offset) == expected + + +@pytest.mark.parametrize( + 'offset', + [BaseCFTimeOffset(), MonthBegin(), YearBegin()], + ids=_id_func +) +def test_to_offset_offset_input(offset): + assert to_offset(offset) == offset + + +@pytest.mark.parametrize( + ('freq', 'expected'), + [('M', MonthEnd()), + ('2M', MonthEnd(n=2)), + ('MS', MonthBegin()), + ('2MS', MonthBegin(n=2)), + ('D', Day()), + ('2D', Day(n=2)), + ('H', Hour()), + ('2H', Hour(n=2)), + ('T', Minute()), + ('2T', Minute(n=2)), + ('min', Minute()), + ('2min', Minute(n=2)), + ('S', Second()), + ('2S', Second(n=2))], + ids=_id_func +) +def test_to_offset_sub_annual(freq, expected): + assert to_offset(freq) == expected + + +_ANNUAL_OFFSET_TYPES = { + 'A': YearEnd, + 'AS': YearBegin +} + + +@pytest.mark.parametrize(('month_int', 'month_label'), + list(_MONTH_ABBREVIATIONS.items()) + [('', '')]) +@pytest.mark.parametrize('multiple', [None, 2]) +@pytest.mark.parametrize('offset_str', ['AS', 'A']) +def test_to_offset_annual(month_label, month_int, multiple, offset_str): + freq = offset_str + offset_type = _ANNUAL_OFFSET_TYPES[offset_str] + if month_label: + freq = '-'.join([freq, month_label]) + if multiple: + freq = '{}'.format(multiple) + freq + result = to_offset(freq) + + if multiple and month_int: + expected = offset_type(n=multiple, month=month_int) + elif multiple: + expected = offset_type(n=multiple) + elif month_int: + expected = offset_type(month=month_int) + else: + expected = offset_type() + assert result == expected + + +@pytest.mark.parametrize('freq', ['Z', '7min2', 'AM', 'M-', 'AS-', '1H1min']) +def test_invalid_to_offset_str(freq): + with pytest.raises(ValueError): + to_offset(freq) + + +@pytest.mark.parametrize( + ('argument', 'expected_date_args'), + [('2000-01-01', (2000, 1, 1)), + ((2000, 1, 1), (2000, 1, 1))], + ids=_id_func +) +def test_to_cftime_datetime(calendar, argument, expected_date_args): + date_type = get_date_type(calendar) + expected = date_type(*expected_date_args) + if isinstance(argument, tuple): + argument = date_type(*argument) + result = to_cftime_datetime(argument, calendar=calendar) + assert result == expected + + +def test_to_cftime_datetime_error_no_calendar(): + with pytest.raises(ValueError): + to_cftime_datetime('2000') + + +def test_to_cftime_datetime_error_type_error(): + with pytest.raises(TypeError): + to_cftime_datetime(1) + + +_EQ_TESTS_A = [ + BaseCFTimeOffset(), YearBegin(), YearEnd(), YearBegin(month=2), + YearEnd(month=2), MonthBegin(), MonthEnd(), Day(), Hour(), Minute(), + Second() +] +_EQ_TESTS_B = [ + BaseCFTimeOffset(n=2), YearBegin(n=2), YearEnd(n=2), + YearBegin(n=2, month=2), YearEnd(n=2, month=2), MonthBegin(n=2), + MonthEnd(n=2), Day(n=2), Hour(n=2), Minute(n=2), Second(n=2) +] + + +@pytest.mark.parametrize( + ('a', 'b'), product(_EQ_TESTS_A, _EQ_TESTS_B), ids=_id_func +) +def test_neq(a, b): + assert a != b + + +_EQ_TESTS_B_COPY = [ + BaseCFTimeOffset(n=2), YearBegin(n=2), YearEnd(n=2), + YearBegin(n=2, month=2), YearEnd(n=2, month=2), MonthBegin(n=2), + MonthEnd(n=2), Day(n=2), Hour(n=2), Minute(n=2), Second(n=2) +] + + +@pytest.mark.parametrize( + ('a', 'b'), zip(_EQ_TESTS_B, _EQ_TESTS_B_COPY), ids=_id_func +) +def test_eq(a, b): + assert a == b + + +_MUL_TESTS = [ + (BaseCFTimeOffset(), BaseCFTimeOffset(n=3)), + (YearEnd(), YearEnd(n=3)), + (YearBegin(), YearBegin(n=3)), + (MonthEnd(), MonthEnd(n=3)), + (MonthBegin(), MonthBegin(n=3)), + (Day(), Day(n=3)), + (Hour(), Hour(n=3)), + (Minute(), Minute(n=3)), + (Second(), Second(n=3)) +] + + +@pytest.mark.parametrize(('offset', 'expected'), _MUL_TESTS, ids=_id_func) +def test_mul(offset, expected): + assert offset * 3 == expected + + +@pytest.mark.parametrize(('offset', 'expected'), _MUL_TESTS, ids=_id_func) +def test_rmul(offset, expected): + assert 3 * offset == expected + + +@pytest.mark.parametrize( + ('offset', 'expected'), + [(BaseCFTimeOffset(), BaseCFTimeOffset(n=-1)), + (YearEnd(), YearEnd(n=-1)), + (YearBegin(), YearBegin(n=-1)), + (MonthEnd(), MonthEnd(n=-1)), + (MonthBegin(), MonthBegin(n=-1)), + (Day(), Day(n=-1)), + (Hour(), Hour(n=-1)), + (Minute(), Minute(n=-1)), + (Second(), Second(n=-1))], + ids=_id_func) +def test_neg(offset, expected): + assert -offset == expected + + +_ADD_TESTS = [ + (Day(n=2), (1, 1, 3)), + (Hour(n=2), (1, 1, 1, 2)), + (Minute(n=2), (1, 1, 1, 0, 2)), + (Second(n=2), (1, 1, 1, 0, 0, 2)) +] + + +@pytest.mark.parametrize( + ('offset', 'expected_date_args'), + _ADD_TESTS, + ids=_id_func +) +def test_add_sub_monthly(offset, expected_date_args, calendar): + date_type = get_date_type(calendar) + initial = date_type(1, 1, 1) + expected = date_type(*expected_date_args) + result = offset + initial + assert result == expected + + +@pytest.mark.parametrize( + ('offset', 'expected_date_args'), + _ADD_TESTS, + ids=_id_func +) +def test_radd_sub_monthly(offset, expected_date_args, calendar): + date_type = get_date_type(calendar) + initial = date_type(1, 1, 1) + expected = date_type(*expected_date_args) + result = initial + offset + assert result == expected + + +@pytest.mark.parametrize( + ('offset', 'expected_date_args'), + [(Day(n=2), (1, 1, 1)), + (Hour(n=2), (1, 1, 2, 22)), + (Minute(n=2), (1, 1, 2, 23, 58)), + (Second(n=2), (1, 1, 2, 23, 59, 58))], + ids=_id_func +) +def test_rsub_sub_monthly(offset, expected_date_args, calendar): + date_type = get_date_type(calendar) + initial = date_type(1, 1, 3) + expected = date_type(*expected_date_args) + result = initial - offset + assert result == expected + + +@pytest.mark.parametrize('offset', _EQ_TESTS_A, ids=_id_func) +def test_sub_error(offset, calendar): + date_type = get_date_type(calendar) + initial = date_type(1, 1, 1) + with pytest.raises(TypeError): + offset - initial + + +@pytest.mark.parametrize( + ('a', 'b'), + zip(_EQ_TESTS_A, _EQ_TESTS_B), + ids=_id_func +) +def test_minus_offset(a, b): + result = b - a + expected = a + assert result == expected + + +@pytest.mark.parametrize( + ('a', 'b'), + list(zip(np.roll(_EQ_TESTS_A, 1), _EQ_TESTS_B)) + + [(YearEnd(month=1), YearEnd(month=2))], + ids=_id_func +) +def test_minus_offset_error(a, b): + with pytest.raises(TypeError): + b - a + + +def test_days_in_month_non_december(calendar): + date_type = get_date_type(calendar) + reference = date_type(1, 4, 1) + assert _days_in_month(reference) == 30 + + +def test_days_in_month_december(calendar): + if calendar == '360_day': + expected = 30 + else: + expected = 31 + date_type = get_date_type(calendar) + reference = date_type(1, 12, 5) + assert _days_in_month(reference) == expected + + +@pytest.mark.parametrize( + ('initial_date_args', 'offset', 'expected_date_args'), + [((1, 1, 1), MonthBegin(), (1, 2, 1)), + ((1, 1, 1), MonthBegin(n=2), (1, 3, 1)), + ((1, 1, 7), MonthBegin(), (1, 2, 1)), + ((1, 1, 7), MonthBegin(n=2), (1, 3, 1)), + ((1, 3, 1), MonthBegin(n=-1), (1, 2, 1)), + ((1, 3, 1), MonthBegin(n=-2), (1, 1, 1)), + ((1, 3, 3), MonthBegin(n=-1), (1, 3, 1)), + ((1, 3, 3), MonthBegin(n=-2), (1, 2, 1)), + ((1, 2, 1), MonthBegin(n=14), (2, 4, 1)), + ((2, 4, 1), MonthBegin(n=-14), (1, 2, 1)), + ((1, 1, 1, 5, 5, 5, 5), MonthBegin(), (1, 2, 1, 5, 5, 5, 5)), + ((1, 1, 3, 5, 5, 5, 5), MonthBegin(), (1, 2, 1, 5, 5, 5, 5)), + ((1, 1, 3, 5, 5, 5, 5), MonthBegin(n=-1), (1, 1, 1, 5, 5, 5, 5))], + ids=_id_func +) +def test_add_month_begin( + calendar, initial_date_args, offset, expected_date_args): + date_type = get_date_type(calendar) + initial = date_type(*initial_date_args) + result = initial + offset + expected = date_type(*expected_date_args) + assert result == expected + + +@pytest.mark.parametrize( + ('initial_date_args', 'offset', 'expected_year_month', + 'expected_sub_day'), + [((1, 1, 1), MonthEnd(), (1, 1), ()), + ((1, 1, 1), MonthEnd(n=2), (1, 2), ()), + ((1, 3, 1), MonthEnd(n=-1), (1, 2), ()), + ((1, 3, 1), MonthEnd(n=-2), (1, 1), ()), + ((1, 2, 1), MonthEnd(n=14), (2, 3), ()), + ((2, 4, 1), MonthEnd(n=-14), (1, 2), ()), + ((1, 1, 1, 5, 5, 5, 5), MonthEnd(), (1, 1), (5, 5, 5, 5)), + ((1, 2, 1, 5, 5, 5, 5), MonthEnd(n=-1), (1, 1), (5, 5, 5, 5))], + ids=_id_func +) +def test_add_month_end( + calendar, initial_date_args, offset, expected_year_month, + expected_sub_day +): + date_type = get_date_type(calendar) + initial = date_type(*initial_date_args) + result = initial + offset + reference_args = expected_year_month + (1,) + reference = date_type(*reference_args) + + # Here the days at the end of each month varies based on the calendar used + expected_date_args = (expected_year_month + + (_days_in_month(reference),) + expected_sub_day) + expected = date_type(*expected_date_args) + assert result == expected + + +@pytest.mark.parametrize( + ('initial_year_month', 'initial_sub_day', 'offset', 'expected_year_month', + 'expected_sub_day'), + [((1, 1), (), MonthEnd(), (1, 2), ()), + ((1, 1), (), MonthEnd(n=2), (1, 3), ()), + ((1, 3), (), MonthEnd(n=-1), (1, 2), ()), + ((1, 3), (), MonthEnd(n=-2), (1, 1), ()), + ((1, 2), (), MonthEnd(n=14), (2, 4), ()), + ((2, 4), (), MonthEnd(n=-14), (1, 2), ()), + ((1, 1), (5, 5, 5, 5), MonthEnd(), (1, 2), (5, 5, 5, 5)), + ((1, 2), (5, 5, 5, 5), MonthEnd(n=-1), (1, 1), (5, 5, 5, 5))], + ids=_id_func +) +def test_add_month_end_onOffset( + calendar, initial_year_month, initial_sub_day, offset, expected_year_month, + expected_sub_day +): + date_type = get_date_type(calendar) + reference_args = initial_year_month + (1,) + reference = date_type(*reference_args) + initial_date_args = (initial_year_month + (_days_in_month(reference),) + + initial_sub_day) + initial = date_type(*initial_date_args) + result = initial + offset + reference_args = expected_year_month + (1,) + reference = date_type(*reference_args) + + # Here the days at the end of each month varies based on the calendar used + expected_date_args = (expected_year_month + + (_days_in_month(reference),) + expected_sub_day) + expected = date_type(*expected_date_args) + assert result == expected + + +@pytest.mark.parametrize( + ('initial_date_args', 'offset', 'expected_date_args'), + [((1, 1, 1), YearBegin(), (2, 1, 1)), + ((1, 1, 1), YearBegin(n=2), (3, 1, 1)), + ((1, 1, 1), YearBegin(month=2), (1, 2, 1)), + ((1, 1, 7), YearBegin(n=2), (3, 1, 1)), + ((2, 2, 1), YearBegin(n=-1), (2, 1, 1)), + ((1, 1, 2), YearBegin(n=-1), (1, 1, 1)), + ((1, 1, 1, 5, 5, 5, 5), YearBegin(), (2, 1, 1, 5, 5, 5, 5)), + ((2, 1, 1, 5, 5, 5, 5), YearBegin(n=-1), (1, 1, 1, 5, 5, 5, 5))], + ids=_id_func +) +def test_add_year_begin(calendar, initial_date_args, offset, + expected_date_args): + date_type = get_date_type(calendar) + initial = date_type(*initial_date_args) + result = initial + offset + expected = date_type(*expected_date_args) + assert result == expected + + +@pytest.mark.parametrize( + ('initial_date_args', 'offset', 'expected_year_month', + 'expected_sub_day'), + [((1, 1, 1), YearEnd(), (1, 12), ()), + ((1, 1, 1), YearEnd(n=2), (2, 12), ()), + ((1, 1, 1), YearEnd(month=1), (1, 1), ()), + ((2, 3, 1), YearEnd(n=-1), (1, 12), ()), + ((1, 3, 1), YearEnd(n=-1, month=2), (1, 2), ()), + ((1, 1, 1, 5, 5, 5, 5), YearEnd(), (1, 12), (5, 5, 5, 5)), + ((1, 1, 1, 5, 5, 5, 5), YearEnd(n=2), (2, 12), (5, 5, 5, 5))], + ids=_id_func +) +def test_add_year_end( + calendar, initial_date_args, offset, expected_year_month, + expected_sub_day +): + date_type = get_date_type(calendar) + initial = date_type(*initial_date_args) + result = initial + offset + reference_args = expected_year_month + (1,) + reference = date_type(*reference_args) + + # Here the days at the end of each month varies based on the calendar used + expected_date_args = (expected_year_month + + (_days_in_month(reference),) + expected_sub_day) + expected = date_type(*expected_date_args) + assert result == expected + + +@pytest.mark.parametrize( + ('initial_year_month', 'initial_sub_day', 'offset', 'expected_year_month', + 'expected_sub_day'), + [((1, 12), (), YearEnd(), (2, 12), ()), + ((1, 12), (), YearEnd(n=2), (3, 12), ()), + ((2, 12), (), YearEnd(n=-1), (1, 12), ()), + ((3, 12), (), YearEnd(n=-2), (1, 12), ()), + ((1, 1), (), YearEnd(month=2), (1, 2), ()), + ((1, 12), (5, 5, 5, 5), YearEnd(), (2, 12), (5, 5, 5, 5)), + ((2, 12), (5, 5, 5, 5), YearEnd(n=-1), (1, 12), (5, 5, 5, 5))], + ids=_id_func +) +def test_add_year_end_onOffset( + calendar, initial_year_month, initial_sub_day, offset, expected_year_month, + expected_sub_day +): + date_type = get_date_type(calendar) + reference_args = initial_year_month + (1,) + reference = date_type(*reference_args) + initial_date_args = (initial_year_month + (_days_in_month(reference),) + + initial_sub_day) + initial = date_type(*initial_date_args) + result = initial + offset + reference_args = expected_year_month + (1,) + reference = date_type(*reference_args) + + # Here the days at the end of each month varies based on the calendar used + expected_date_args = (expected_year_month + + (_days_in_month(reference),) + expected_sub_day) + expected = date_type(*expected_date_args) + assert result == expected + + +# Note for all sub-monthly offsets, pandas always returns True for onOffset +@pytest.mark.parametrize( + ('date_args', 'offset', 'expected'), + [((1, 1, 1), MonthBegin(), True), + ((1, 1, 1, 1), MonthBegin(), True), + ((1, 1, 5), MonthBegin(), False), + ((1, 1, 5), MonthEnd(), False), + ((1, 1, 1), YearBegin(), True), + ((1, 1, 1, 1), YearBegin(), True), + ((1, 1, 5), YearBegin(), False), + ((1, 12, 1), YearEnd(), False), + ((1, 1, 1), Day(), True), + ((1, 1, 1, 1), Day(), True), + ((1, 1, 1), Hour(), True), + ((1, 1, 1), Minute(), True), + ((1, 1, 1), Second(), True)], + ids=_id_func +) +def test_onOffset(calendar, date_args, offset, expected): + date_type = get_date_type(calendar) + date = date_type(*date_args) + result = offset.onOffset(date) + assert result == expected + + +@pytest.mark.parametrize( + ('year_month_args', 'sub_day_args', 'offset'), + [((1, 1), (), MonthEnd()), + ((1, 1), (1,), MonthEnd()), + ((1, 12), (), YearEnd()), + ((1, 1), (), YearEnd(month=1))], + ids=_id_func +) +def test_onOffset_month_or_year_end( + calendar, year_month_args, sub_day_args, offset): + date_type = get_date_type(calendar) + reference_args = year_month_args + (1,) + reference = date_type(*reference_args) + date_args = year_month_args + (_days_in_month(reference),) + sub_day_args + date = date_type(*date_args) + result = offset.onOffset(date) + assert result + + +@pytest.mark.parametrize( + ('offset', 'initial_date_args', 'partial_expected_date_args'), + [(YearBegin(), (1, 3, 1), (2, 1)), + (YearBegin(), (1, 1, 1), (1, 1)), + (YearBegin(n=2), (1, 3, 1), (2, 1)), + (YearBegin(n=2, month=2), (1, 3, 1), (2, 2)), + (YearEnd(), (1, 3, 1), (1, 12)), + (YearEnd(n=2), (1, 3, 1), (1, 12)), + (YearEnd(n=2, month=2), (1, 3, 1), (2, 2)), + (YearEnd(n=2, month=4), (1, 4, 30), (1, 4)), + (MonthBegin(), (1, 3, 2), (1, 4)), + (MonthBegin(), (1, 3, 1), (1, 3)), + (MonthBegin(n=2), (1, 3, 2), (1, 4)), + (MonthEnd(), (1, 3, 2), (1, 3)), + (MonthEnd(), (1, 4, 30), (1, 4)), + (MonthEnd(n=2), (1, 3, 2), (1, 3)), + (Day(), (1, 3, 2, 1), (1, 3, 2, 1)), + (Hour(), (1, 3, 2, 1, 1), (1, 3, 2, 1, 1)), + (Minute(), (1, 3, 2, 1, 1, 1), (1, 3, 2, 1, 1, 1)), + (Second(), (1, 3, 2, 1, 1, 1, 1), (1, 3, 2, 1, 1, 1, 1))], + ids=_id_func +) +def test_rollforward(calendar, offset, initial_date_args, + partial_expected_date_args): + date_type = get_date_type(calendar) + initial = date_type(*initial_date_args) + if isinstance(offset, (MonthBegin, YearBegin)): + expected_date_args = partial_expected_date_args + (1,) + elif isinstance(offset, (MonthEnd, YearEnd)): + reference_args = partial_expected_date_args + (1,) + reference = date_type(*reference_args) + expected_date_args = (partial_expected_date_args + + (_days_in_month(reference),)) + else: + expected_date_args = partial_expected_date_args + expected = date_type(*expected_date_args) + result = offset.rollforward(initial) + assert result == expected + + +@pytest.mark.parametrize( + ('offset', 'initial_date_args', 'partial_expected_date_args'), + [(YearBegin(), (1, 3, 1), (1, 1)), + (YearBegin(n=2), (1, 3, 1), (1, 1)), + (YearBegin(n=2, month=2), (1, 3, 1), (1, 2)), + (YearBegin(), (1, 1, 1), (1, 1)), + (YearBegin(n=2, month=2), (1, 2, 1), (1, 2)), + (YearEnd(), (2, 3, 1), (1, 12)), + (YearEnd(n=2), (2, 3, 1), (1, 12)), + (YearEnd(n=2, month=2), (2, 3, 1), (2, 2)), + (YearEnd(month=4), (1, 4, 30), (1, 4)), + (MonthBegin(), (1, 3, 2), (1, 3)), + (MonthBegin(n=2), (1, 3, 2), (1, 3)), + (MonthBegin(), (1, 3, 1), (1, 3)), + (MonthEnd(), (1, 3, 2), (1, 2)), + (MonthEnd(n=2), (1, 3, 2), (1, 2)), + (MonthEnd(), (1, 4, 30), (1, 4)), + (Day(), (1, 3, 2, 1), (1, 3, 2, 1)), + (Hour(), (1, 3, 2, 1, 1), (1, 3, 2, 1, 1)), + (Minute(), (1, 3, 2, 1, 1, 1), (1, 3, 2, 1, 1, 1)), + (Second(), (1, 3, 2, 1, 1, 1, 1), (1, 3, 2, 1, 1, 1, 1))], + ids=_id_func +) +def test_rollback(calendar, offset, initial_date_args, + partial_expected_date_args): + date_type = get_date_type(calendar) + initial = date_type(*initial_date_args) + if isinstance(offset, (MonthBegin, YearBegin)): + expected_date_args = partial_expected_date_args + (1,) + elif isinstance(offset, (MonthEnd, YearEnd)): + reference_args = partial_expected_date_args + (1,) + reference = date_type(*reference_args) + expected_date_args = (partial_expected_date_args + + (_days_in_month(reference),)) + else: + expected_date_args = partial_expected_date_args + expected = date_type(*expected_date_args) + result = offset.rollback(initial) + assert result == expected + + +_CFTIME_RANGE_TESTS = [ + ('0001-01-01', '0001-01-04', None, 'D', None, False, + [(1, 1, 1), (1, 1, 2), (1, 1, 3), (1, 1, 4)]), + ('0001-01-01', '0001-01-04', None, 'D', 'left', False, + [(1, 1, 1), (1, 1, 2), (1, 1, 3)]), + ('0001-01-01', '0001-01-04', None, 'D', 'right', False, + [(1, 1, 2), (1, 1, 3), (1, 1, 4)]), + ('0001-01-01T01:00:00', '0001-01-04', None, 'D', None, False, + [(1, 1, 1, 1), (1, 1, 2, 1), (1, 1, 3, 1)]), + ('0001-01-01T01:00:00', '0001-01-04', None, 'D', None, True, + [(1, 1, 1), (1, 1, 2), (1, 1, 3), (1, 1, 4)]), + ('0001-01-01', None, 4, 'D', None, False, + [(1, 1, 1), (1, 1, 2), (1, 1, 3), (1, 1, 4)]), + (None, '0001-01-04', 4, 'D', None, False, + [(1, 1, 1), (1, 1, 2), (1, 1, 3), (1, 1, 4)]), + ((1, 1, 1), '0001-01-04', None, 'D', None, False, + [(1, 1, 1), (1, 1, 2), (1, 1, 3), (1, 1, 4)]), + ((1, 1, 1), (1, 1, 4), None, 'D', None, False, + [(1, 1, 1), (1, 1, 2), (1, 1, 3), (1, 1, 4)]), + ('0001-01-30', '0011-02-01', None, '3AS-JUN', None, False, + [(1, 6, 1), (4, 6, 1), (7, 6, 1), (10, 6, 1)]), + ('0001-01-04', '0001-01-01', None, 'D', None, False, + []), + ('0010', None, 4, YearBegin(n=-2), None, False, + [(10, 1, 1), (8, 1, 1), (6, 1, 1), (4, 1, 1)]), + ('0001-01-01', '0001-01-04', 4, None, None, False, + [(1, 1, 1), (1, 1, 2), (1, 1, 3), (1, 1, 4)]) +] + + +@pytest.mark.parametrize( + ('start', 'end', 'periods', 'freq', 'closed', 'normalize', + 'expected_date_args'), + _CFTIME_RANGE_TESTS, ids=_id_func +) +def test_cftime_range( + start, end, periods, freq, closed, normalize, calendar, + expected_date_args): + date_type = get_date_type(calendar) + expected_dates = [date_type(*args) for args in expected_date_args] + + if isinstance(start, tuple): + start = date_type(*start) + if isinstance(end, tuple): + end = date_type(*end) + + result = cftime_range( + start=start, end=end, periods=periods, freq=freq, closed=closed, + normalize=normalize, calendar=calendar) + resulting_dates = result.values + + assert isinstance(result, CFTimeIndex) + + if freq is not None: + np.testing.assert_equal(resulting_dates, expected_dates) + else: + # If we create a linear range of dates using cftime.num2date + # we will not get exact round number dates. This is because + # datetime arithmetic in cftime is accurate approximately to + # 1 millisecond (see https://unidata.github.io/cftime/api.html). + deltas = resulting_dates - expected_dates + deltas = np.array([delta.total_seconds() for delta in deltas]) + assert np.max(np.abs(deltas)) < 0.001 + + +def test_cftime_range_name(): + result = cftime_range(start='2000', periods=4, name='foo') + assert result.name == 'foo' + + result = cftime_range(start='2000', periods=4) + assert result.name is None + + +@pytest.mark.parametrize( + ('start', 'end', 'periods', 'freq', 'closed'), + [(None, None, 5, 'A', None), + ('2000', None, None, 'A', None), + (None, '2000', None, 'A', None), + ('2000', '2001', None, None, None), + (None, None, None, None, None), + ('2000', '2001', None, 'A', 'up'), + ('2000', '2001', 5, 'A', None)] +) +def test_invalid_cftime_range_inputs(start, end, periods, freq, closed): + with pytest.raises(ValueError): + cftime_range(start, end, periods, freq, closed=closed) + + +_CALENDAR_SPECIFIC_MONTH_END_TESTS = [ + ('2M', 'noleap', + [(2, 28), (4, 30), (6, 30), (8, 31), (10, 31), (12, 31)]), + ('2M', 'all_leap', + [(2, 29), (4, 30), (6, 30), (8, 31), (10, 31), (12, 31)]), + ('2M', '360_day', + [(2, 30), (4, 30), (6, 30), (8, 30), (10, 30), (12, 30)]), + ('2M', 'standard', + [(2, 29), (4, 30), (6, 30), (8, 31), (10, 31), (12, 31)]), + ('2M', 'gregorian', + [(2, 29), (4, 30), (6, 30), (8, 31), (10, 31), (12, 31)]), + ('2M', 'julian', + [(2, 29), (4, 30), (6, 30), (8, 31), (10, 31), (12, 31)]) +] + + +@pytest.mark.parametrize( + ('freq', 'calendar', 'expected_month_day'), + _CALENDAR_SPECIFIC_MONTH_END_TESTS, ids=_id_func +) +def test_calendar_specific_month_end(freq, calendar, expected_month_day): + year = 2000 # Use a leap-year to highlight calendar differences + result = cftime_range( + start='2000-02', end='2001', freq=freq, calendar=calendar).values + date_type = get_date_type(calendar) + expected = [date_type(year, *args) for args in expected_month_day] + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize( + ('calendar', 'start', 'end', 'expected_number_of_days'), + [('noleap', '2000', '2001', 365), + ('all_leap', '2000', '2001', 366), + ('360_day', '2000', '2001', 360), + ('standard', '2000', '2001', 366), + ('gregorian', '2000', '2001', 366), + ('julian', '2000', '2001', 366), + ('noleap', '2001', '2002', 365), + ('all_leap', '2001', '2002', 366), + ('360_day', '2001', '2002', 360), + ('standard', '2001', '2002', 365), + ('gregorian', '2001', '2002', 365), + ('julian', '2001', '2002', 365)] +) +def test_calendar_year_length( + calendar, start, end, expected_number_of_days): + result = cftime_range(start, end, freq='D', closed='left', + calendar=calendar) + assert len(result) == expected_number_of_days diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index 6f102b60b9d..f72c6904f0e 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -2,6 +2,7 @@ import pytest +import numpy as np import pandas as pd import xarray as xr @@ -121,22 +122,42 @@ def dec_days(date_type): return 31 +@pytest.fixture +def index_with_name(date_type): + dates = [date_type(1, 1, 1), date_type(1, 2, 1), + date_type(2, 1, 1), date_type(2, 2, 1)] + return CFTimeIndex(dates, name='foo') + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize( + ('name', 'expected_name'), + [('bar', 'bar'), + (None, 'foo')]) +def test_constructor_with_name(index_with_name, name, expected_name): + result = CFTimeIndex(index_with_name, name=name).name + assert result == expected_name + + @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_assert_all_valid_date_type(date_type, index): import cftime if date_type is cftime.DatetimeNoLeap: - mixed_date_types = [date_type(1, 1, 1), - cftime.DatetimeAllLeap(1, 2, 1)] + mixed_date_types = np.array( + [date_type(1, 1, 1), + cftime.DatetimeAllLeap(1, 2, 1)]) else: - mixed_date_types = [date_type(1, 1, 1), - cftime.DatetimeNoLeap(1, 2, 1)] + mixed_date_types = np.array( + [date_type(1, 1, 1), + cftime.DatetimeNoLeap(1, 2, 1)]) with pytest.raises(TypeError): assert_all_valid_date_type(mixed_date_types) with pytest.raises(TypeError): - assert_all_valid_date_type([1, date_type(1, 1, 1)]) + assert_all_valid_date_type(np.array([1, date_type(1, 1, 1)])) - assert_all_valid_date_type([date_type(1, 1, 1), date_type(1, 2, 1)]) + assert_all_valid_date_type( + np.array([date_type(1, 1, 1), date_type(1, 2, 1)])) @pytest.mark.skipif(not has_cftime, reason='cftime not installed') @@ -589,3 +610,9 @@ def test_concat_cftimeindex(date_type, enable_cftimeindex): else: assert isinstance(da.indexes['time'], pd.Index) assert not isinstance(da.indexes['time'], CFTimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_empty_cftimeindex(): + index = CFTimeIndex([]) + assert index.date_type is None From c1c576f75a2c4c2f8fad314c10588f45c5a5c573 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Fri, 21 Sep 2018 19:36:20 +0200 Subject: [PATCH 217/282] Plotting: restore xyincrease kwarg default to True (#2425) --- xarray/plot/plot.py | 8 ++++++-- xarray/tests/test_plot.py | 22 ++++++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 10fca44b417..b92429b857d 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -624,7 +624,7 @@ def _plot2d(plotfunc): @functools.wraps(plotfunc) def newplotfunc(darray, x=None, y=None, figsize=None, size=None, aspect=None, ax=None, row=None, col=None, - col_wrap=None, xincrease=None, yincrease=None, + col_wrap=None, xincrease=True, yincrease=True, add_colorbar=None, add_labels=True, vmin=None, vmax=None, cmap=None, center=None, robust=False, extend=None, levels=None, infer_intervals=None, colors=None, @@ -776,6 +776,10 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, raise ValueError("cbar_ax and cbar_kwargs can't be used with " "add_colorbar=False.") + # origin kwarg overrides yincrease + if 'origin' in kwargs: + yincrease = None + _update_axes(ax, xincrease, yincrease, xscale, yscale, xticks, yticks, xlim, ylim) @@ -794,7 +798,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, @functools.wraps(newplotfunc) def plotmethod(_PlotMethods_obj, x=None, y=None, figsize=None, size=None, aspect=None, ax=None, row=None, col=None, col_wrap=None, - xincrease=None, yincrease=None, add_colorbar=None, + xincrease=True, yincrease=True, add_colorbar=None, add_labels=True, vmin=None, vmax=None, cmap=None, colors=None, center=None, robust=False, extend=None, levels=None, infer_intervals=None, subplot_kws=None, diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 15cb6af5fb1..e27f03630b7 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -762,6 +762,24 @@ def test_nonnumeric_index_raises_typeerror(self): def test_can_pass_in_axis(self): self.pass_in_axis(self.plotmethod) + def test_xyincrease_defaults(self): + + # With default settings the axis must be ordered regardless + # of the coords order. + self.plotfunc(DataArray(easy_array((3, 2)), coords=[[1, 2, 3], + [1, 2]])) + bounds = plt.gca().get_ylim() + assert bounds[0] < bounds[1] + bounds = plt.gca().get_xlim() + assert bounds[0] < bounds[1] + # Inverted coords + self.plotfunc(DataArray(easy_array((3, 2)), coords=[[3, 2, 1], + [2, 1]])) + bounds = plt.gca().get_ylim() + assert bounds[0] < bounds[1] + bounds = plt.gca().get_xlim() + assert bounds[0] < bounds[1] + def test_xyincrease_false_changes_axes(self): self.plotmethod(xincrease=False, yincrease=False) xlim = plt.gca().get_xlim() @@ -1308,8 +1326,8 @@ def test_regression_rgb_imshow_dim_size_one(self): da = DataArray(easy_array((1, 3, 3), start=0.0, stop=1.0)) da.plot.imshow() - def test_imshow_origin_kwarg(self): - da = DataArray(easy_array((5, 5, 3), start=-0.6, stop=1.4)) + def test_origin_overrides_xyincrease(self): + da = DataArray(easy_array((3, 2)), coords=[[-2, 0, 2], [-1, 1]]) da.plot.imshow(origin='upper') assert plt.xlim()[0] < 0 assert plt.ylim()[1] < 0 From ab96954883200f764a0dd50870e4db240c119265 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sat, 22 Sep 2018 05:02:42 +0900 Subject: [PATCH 218/282] implement Gradient (#2398) * Added xr.gradient, DataArray.gradient, Dataset.gradient * Working with np.backend * test is not passing * Docs * flake8 * support environment without dask * Support numpy < 1.13 * Support numpy 1.12 * simplify dask.gradient * lint * Use npcompat.gradient in tests * move gradient to dask_array_compat * gradient -> differentiate * lint * Update dask_array_compat * Added a link from diff * remove xr.differentiate * Added datetime support * Update via comment. Use utils.to_numeric also in interp * time_unit -> datetime_unit * Some more info in docs. * update test * Update via comments * Update docs. --- doc/api.rst | 2 + doc/computation.rst | 23 ++++ doc/whats-new.rst | 4 + xarray/__init__.py | 2 +- xarray/core/dask_array_compat.py | 124 +++++++++++++++++++ xarray/core/dataarray.py | 58 +++++++++ xarray/core/dataset.py | 65 +++++++++- xarray/core/duck_array_ops.py | 8 ++ xarray/core/missing.py | 6 +- xarray/core/npcompat.py | 185 ++++++++++++++++++++++++++++ xarray/core/utils.py | 21 ++++ xarray/tests/test_dataset.py | 77 +++++++++++- xarray/tests/test_duck_array_ops.py | 21 +++- 13 files changed, 586 insertions(+), 10 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 89fee10506d..d204fab3539 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -150,6 +150,7 @@ Computation Dataset.resample Dataset.diff Dataset.quantile + Dataset.differentiate **Aggregation**: :py:attr:`~Dataset.all` @@ -317,6 +318,7 @@ Computation DataArray.diff DataArray.dot DataArray.quantile + DataArray.differentiate **Aggregation**: :py:attr:`~DataArray.all` diff --git a/doc/computation.rst b/doc/computation.rst index 6793e667e06..67cda6f2191 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -200,6 +200,29 @@ You can also use ``construct`` to compute a weighted rolling sum: To avoid this, use ``skipna=False`` as the above example. +Computation using Coordinates +============================= + +Xarray objects have some handy methods for the computation with their +coordinates. :py:meth:`~xarray.DataArray.differentiate` computes derivatives by +central finite differences using their coordinates, + +.. ipython:: python + a = xr.DataArray([0, 1, 2, 3], dims=['x'], coords=[0.1, 0.11, 0.2, 0.3]) + a + a.differentiate('x') + +This method can be used also for multidimensional arrays, + +.. ipython:: python + a = xr.DataArray(np.arange(8).reshape(4, 2), dims=['x', 'y'], + coords=[0.1, 0.11, 0.2, 0.3]) + a.differentiate('x') + +.. note:: + This method is limited to simple cartesian geometry. Differentiation along + multidimensional coordinate is not supported. + .. _compute.broadcasting: Broadcasting by dimension name diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8c34ddf3fa9..7240059bd10 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,10 @@ Documentation Enhancements ~~~~~~~~~~~~ +- :py:meth:`~xarray.DataArray.differentiate` and + :py:meth:`~xarray.Dataset.differentiate` are newly added. + (:issue:`1332`) + By `Keisuke Fujii `_. - Default colormap for sequential and divergent data can now be set via :py:func:`~xarray.set_options()` (:issue:`2394`) diff --git a/xarray/__init__.py b/xarray/__init__.py index e2d24e6c294..e3898f348cc 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -10,7 +10,7 @@ from .core.alignment import align, broadcast, broadcast_arrays from .core.common import full_like, zeros_like, ones_like from .core.combine import concat, auto_combine -from .core.computation import apply_ufunc, where, dot +from .core.computation import apply_ufunc, dot, where from .core.extensions import (register_dataarray_accessor, register_dataset_accessor) from .core.variable import as_variable, Variable, IndexVariable, Coordinate diff --git a/xarray/core/dask_array_compat.py b/xarray/core/dask_array_compat.py index c2417345f55..5e6b81a253d 100644 --- a/xarray/core/dask_array_compat.py +++ b/xarray/core/dask_array_compat.py @@ -1,6 +1,9 @@ from __future__ import absolute_import, division, print_function +from distutils.version import LooseVersion + import numpy as np +from dask import __version__ as dask_version import dask.array as da try: @@ -30,3 +33,124 @@ def isin(element, test_elements, assume_unique=False, invert=False): if invert: result = ~result return result + + +if LooseVersion(dask_version) > LooseVersion('1.19.2'): + gradient = da.gradient + +else: # pragma: no cover + # Copied from dask v0.19.2 + # Used under the terms of Dask's license, see licenses/DASK_LICENSE. + import math + from numbers import Integral, Real + + AxisError = np.AxisError + + def validate_axis(axis, ndim): + """ Validate an input to axis= keywords """ + if isinstance(axis, (tuple, list)): + return tuple(validate_axis(ax, ndim) for ax in axis) + if not isinstance(axis, Integral): + raise TypeError("Axis value must be an integer, got %s" % axis) + if axis < -ndim or axis >= ndim: + raise AxisError("Axis %d is out of bounds for array of dimension " + "%d" % (axis, ndim)) + if axis < 0: + axis += ndim + return axis + + def _gradient_kernel(x, block_id, coord, axis, array_locs, grad_kwargs): + """ + x: nd-array + array of one block + coord: 1d-array or scalar + coordinate along which the gradient is computed. + axis: int + axis along which the gradient is computed + array_locs: + actual location along axis. None if coordinate is scalar + grad_kwargs: + keyword to be passed to np.gradient + """ + block_loc = block_id[axis] + if array_locs is not None: + coord = coord[array_locs[0][block_loc]:array_locs[1][block_loc]] + grad = np.gradient(x, coord, axis=axis, **grad_kwargs) + return grad + + def gradient(f, *varargs, **kwargs): + f = da.asarray(f) + + kwargs["edge_order"] = math.ceil(kwargs.get("edge_order", 1)) + if kwargs["edge_order"] > 2: + raise ValueError("edge_order must be less than or equal to 2.") + + drop_result_list = False + axis = kwargs.pop("axis", None) + if axis is None: + axis = tuple(range(f.ndim)) + elif isinstance(axis, Integral): + drop_result_list = True + axis = (axis,) + + axis = validate_axis(axis, f.ndim) + + if len(axis) != len(set(axis)): + raise ValueError("duplicate axes not allowed") + + axis = tuple(ax % f.ndim for ax in axis) + + if varargs == (): + varargs = (1,) + if len(varargs) == 1: + varargs = len(axis) * varargs + if len(varargs) != len(axis): + raise TypeError( + "Spacing must either be a single scalar, or a scalar / " + "1d-array per axis" + ) + + if issubclass(f.dtype.type, (np.bool8, Integral)): + f = f.astype(float) + elif issubclass(f.dtype.type, Real) and f.dtype.itemsize < 4: + f = f.astype(float) + + results = [] + for i, ax in enumerate(axis): + for c in f.chunks[ax]: + if np.min(c) < kwargs["edge_order"] + 1: + raise ValueError( + 'Chunk size must be larger than edge_order + 1. ' + 'Minimum chunk for aixs {} is {}. Rechunk to ' + 'proceed.'.format(np.min(c), ax)) + + if np.isscalar(varargs[i]): + array_locs = None + else: + if isinstance(varargs[i], da.Array): + raise NotImplementedError( + 'dask array coordinated is not supported.') + # coordinate position for each block taking overlap into + # account + chunk = np.array(f.chunks[ax]) + array_loc_stop = np.cumsum(chunk) + 1 + array_loc_start = array_loc_stop - chunk - 2 + array_loc_stop[-1] -= 1 + array_loc_start[0] = 0 + array_locs = (array_loc_start, array_loc_stop) + + results.append(f.map_overlap( + _gradient_kernel, + dtype=f.dtype, + depth={j: 1 if j == ax else 0 for j in range(f.ndim)}, + boundary="none", + coord=varargs[i], + axis=ax, + array_locs=array_locs, + grad_kwargs=kwargs, + )) + + if drop_result_list: + results = results[0] + + return results diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 937d38d30fa..f131b003a69 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -2073,6 +2073,9 @@ def diff(self, dim, n=1, label='upper'): Coordinates: * x (x) int64 3 4 + See Also + -------- + DataArray.differentiate """ ds = self._to_temp_dataset().diff(n=n, dim=dim, label=label) return self._from_temp_dataset(ds) @@ -2352,6 +2355,61 @@ def rank(self, dim, pct=False, keep_attrs=False): ds = self._to_temp_dataset().rank(dim, pct=pct, keep_attrs=keep_attrs) return self._from_temp_dataset(ds) + def differentiate(self, coord, edge_order=1, datetime_unit=None): + """ Differentiate the array with the second order accurate central + differences. + + .. note:: + This feature is limited to simple cartesian geometry, i.e. coord + must be one dimensional. + + Parameters + ---------- + coord: str + The coordinate to be used to compute the gradient. + edge_order: 1 or 2. Default 1 + N-th order accurate differences at the boundaries. + datetime_unit: None or any of {'Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', + 'us', 'ns', 'ps', 'fs', 'as'} + Unit to compute gradient. Only valid for datetime coordinate. + + Returns + ------- + differentiated: DataArray + + See also + -------- + numpy.gradient: corresponding numpy function + + Examples + -------- + + >>> da = xr.DataArray(np.arange(12).reshape(4, 3), dims=['x', 'y'], + ... coords={'x': [0, 0.1, 1.1, 1.2]}) + >>> da + + array([[ 0, 1, 2], + [ 3, 4, 5], + [ 6, 7, 8], + [ 9, 10, 11]]) + Coordinates: + * x (x) float64 0.0 0.1 1.1 1.2 + Dimensions without coordinates: y + >>> + >>> da.differentiate('x') + + array([[30. , 30. , 30. ], + [27.545455, 27.545455, 27.545455], + [27.545455, 27.545455, 27.545455], + [30. , 30. , 30. ]]) + Coordinates: + * x (x) float64 0.0 0.1 1.1 1.2 + Dimensions without coordinates: y + """ + ds = self._to_temp_dataset().differentiate( + coord, edge_order, datetime_unit) + return self._from_temp_dataset(ds) + # priority most be higher than Variable to properly work with binary ufuncs ops.inject_all_ops_and_reduce_methods(DataArray, priority=60) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 89dba6605a6..9cf304858a6 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -13,8 +13,8 @@ import xarray as xr from . import ( - alignment, duck_array_ops, formatting, groupby, indexing, ops, resample, - rolling, utils) + alignment, computation, duck_array_ops, formatting, groupby, indexing, ops, + resample, rolling, utils) from .. import conventions from .alignment import align from .common import ( @@ -31,7 +31,7 @@ OrderedDict, basestring, dask_array_type, integer_types, iteritems, range) from .utils import ( Frozen, SortedKeysDict, either_dict_or_kwargs, decode_numpy_dict_values, - ensure_us_time_resolution, hashable, maybe_wrap_array) + ensure_us_time_resolution, hashable, maybe_wrap_array, to_numeric) from .variable import IndexVariable, Variable, as_variable, broadcast_variables # list of attributes of pd.DatetimeIndex that are ndarrays of time info @@ -3417,6 +3417,9 @@ def diff(self, dim, n=1, label='upper'): Data variables: foo (x) int64 1 -1 + See Also + -------- + Dataset.differentiate """ if n == 0: return self @@ -3767,6 +3770,62 @@ def rank(self, dim, pct=False, keep_attrs=False): attrs = self.attrs if keep_attrs else None return self._replace_vars_and_dims(variables, coord_names, attrs=attrs) + def differentiate(self, coord, edge_order=1, datetime_unit=None): + """ Differentiate with the second order accurate central + differences. + + .. note:: + This feature is limited to simple cartesian geometry, i.e. coord + must be one dimensional. + + Parameters + ---------- + coord: str + The coordinate to be used to compute the gradient. + edge_order: 1 or 2. Default 1 + N-th order accurate differences at the boundaries. + datetime_unit: None or any of {'Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', + 'us', 'ns', 'ps', 'fs', 'as'} + Unit to compute gradient. Only valid for datetime coordinate. + + Returns + ------- + differentiated: Dataset + + See also + -------- + numpy.gradient: corresponding numpy function + """ + from .variable import Variable + + if coord not in self.variables and coord not in self.dims: + raise ValueError('Coordinate {} does not exist.'.format(coord)) + + coord_var = self[coord].variable + if coord_var.ndim != 1: + raise ValueError('Coordinate {} must be 1 dimensional but is {}' + ' dimensional'.format(coord, coord_var.ndim)) + + dim = coord_var.dims[0] + coord_data = coord_var.data + if coord_data.dtype.kind in 'mM': + if datetime_unit is None: + datetime_unit, _ = np.datetime_data(coord_data.dtype) + coord_data = to_numeric(coord_data, datetime_unit=datetime_unit) + + variables = OrderedDict() + for k, v in self.variables.items(): + if (k in self.data_vars and dim in v.dims and + k not in self.coords): + v = to_numeric(v, datetime_unit=datetime_unit) + grad = duck_array_ops.gradient( + v.data, coord_data, edge_order=edge_order, + axis=v.get_axis_num(dim)) + variables[k] = Variable(v.dims, grad) + else: + variables[k] = v + return self._replace_vars_and_dims(variables) + @property def real(self): return self._unary_op(lambda x: x.real, keep_attrs=True)(self) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 17eb310f8db..ef89dba2ab8 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -93,6 +93,14 @@ def isnull(data): einsum = _dask_or_eager_func('einsum', array_args=slice(1, None), requires_dask='0.17.3') + +def gradient(x, coord, axis, edge_order): + if isinstance(x, dask_array_type): + return dask_array_compat.gradient( + x, coord, axis=axis, edge_order=edge_order) + return npcompat.gradient(x, coord, axis=axis, edge_order=edge_order) + + masked_invalid = _dask_or_eager_func( 'masked_invalid', eager_module=np.ma, dask_module=getattr(dask_array, 'ma', None)) diff --git a/xarray/core/missing.py b/xarray/core/missing.py index 90aa4ffaeda..afb34d99115 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -11,7 +11,7 @@ from . import rolling from .computation import apply_ufunc from .pycompat import iteritems -from .utils import is_scalar, OrderedSet +from .utils import is_scalar, OrderedSet, to_numeric from .variable import Variable, broadcast_variables from .duck_array_ops import dask_array_type @@ -414,8 +414,8 @@ def _floatize_x(x, new_x): # offset (min(x)) and the variation (x - min(x)) can be # represented by float. xmin = np.min(x[i]) - x[i] = (x[i] - xmin).astype(np.float64) - new_x[i] = (new_x[i] - xmin).astype(np.float64) + x[i] = to_numeric(x[i], offset=xmin, dtype=np.float64) + new_x[i] = to_numeric(new_x[i], offset=xmin, dtype=np.float64) return x, new_x diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index 6d4db063b98..22dff44acf8 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, print_function +from distutils.version import LooseVersion import numpy as np try: @@ -97,3 +98,187 @@ def isin(element, test_elements, assume_unique=False, invert=False): element = np.asarray(element) return np.in1d(element, test_elements, assume_unique=assume_unique, invert=invert).reshape(element.shape) + + +if LooseVersion(np.__version__) >= LooseVersion('1.13'): + gradient = np.gradient +else: + def normalize_axis_tuple(axes, N): + if isinstance(axes, int): + axes = (axes, ) + return tuple([N + a if a < 0 else a for a in axes]) + + def gradient(f, *varargs, **kwargs): + f = np.asanyarray(f) + N = f.ndim # number of dimensions + + axes = kwargs.pop('axis', None) + if axes is None: + axes = tuple(range(N)) + else: + axes = normalize_axis_tuple(axes, N) + + len_axes = len(axes) + n = len(varargs) + if n == 0: + # no spacing argument - use 1 in all axes + dx = [1.0] * len_axes + elif n == 1 and np.ndim(varargs[0]) == 0: + # single scalar for all axes + dx = varargs * len_axes + elif n == len_axes: + # scalar or 1d array for each axis + dx = list(varargs) + for i, distances in enumerate(dx): + if np.ndim(distances) == 0: + continue + elif np.ndim(distances) != 1: + raise ValueError("distances must be either scalars or 1d") + if len(distances) != f.shape[axes[i]]: + raise ValueError("when 1d, distances must match the " + "length of the corresponding dimension") + diffx = np.diff(distances) + # if distances are constant reduce to the scalar case + # since it brings a consistent speedup + if (diffx == diffx[0]).all(): + diffx = diffx[0] + dx[i] = diffx + else: + raise TypeError("invalid number of arguments") + + edge_order = kwargs.pop('edge_order', 1) + if kwargs: + raise TypeError('"{}" are not valid keyword arguments.'.format( + '", "'.join(kwargs.keys()))) + if edge_order > 2: + raise ValueError("'edge_order' greater than 2 not supported") + + # use central differences on interior and one-sided differences on the + # endpoints. This preserves second order-accuracy over the full domain. + + outvals = [] + + # create slice objects --- initially all are [:, :, ..., :] + slice1 = [slice(None)] * N + slice2 = [slice(None)] * N + slice3 = [slice(None)] * N + slice4 = [slice(None)] * N + + otype = f.dtype.char + if otype not in ['f', 'd', 'F', 'D', 'm', 'M']: + otype = 'd' + + # Difference of datetime64 elements results in timedelta64 + if otype == 'M': + # Need to use the full dtype name because it contains unit + # information + otype = f.dtype.name.replace('datetime', 'timedelta') + elif otype == 'm': + # Needs to keep the specific units, can't be a general unit + otype = f.dtype + + # Convert datetime64 data into ints. Make dummy variable `y` + # that is a view of ints if the data is datetime64, otherwise + # just set y equal to the array `f`. + if f.dtype.char in ["M", "m"]: + y = f.view('int64') + else: + y = f + + for i, axis in enumerate(axes): + if y.shape[axis] < edge_order + 1: + raise ValueError( + "Shape of array too small to calculate a numerical " + "gradient, at least (edge_order + 1) elements are " + "required.") + # result allocation + out = np.empty_like(y, dtype=otype) + + uniform_spacing = np.ndim(dx[i]) == 0 + + # Numerical differentiation: 2nd order interior + slice1[axis] = slice(1, -1) + slice2[axis] = slice(None, -2) + slice3[axis] = slice(1, -1) + slice4[axis] = slice(2, None) + + if uniform_spacing: + out[slice1] = (f[slice4] - f[slice2]) / (2. * dx[i]) + else: + dx1 = dx[i][0:-1] + dx2 = dx[i][1:] + a = -(dx2) / (dx1 * (dx1 + dx2)) + b = (dx2 - dx1) / (dx1 * dx2) + c = dx1 / (dx2 * (dx1 + dx2)) + # fix the shape for broadcasting + shape = np.ones(N, dtype=int) + shape[axis] = -1 + a.shape = b.shape = c.shape = shape + # 1D equivalent -- + # out[1:-1] = a * f[:-2] + b * f[1:-1] + c * f[2:] + out[slice1] = a * f[slice2] + b * f[slice3] + c * f[slice4] + + # Numerical differentiation: 1st order edges + if edge_order == 1: + slice1[axis] = 0 + slice2[axis] = 1 + slice3[axis] = 0 + dx_0 = dx[i] if uniform_spacing else dx[i][0] + # 1D equivalent -- out[0] = (y[1] - y[0]) / (x[1] - x[0]) + out[slice1] = (y[slice2] - y[slice3]) / dx_0 + + slice1[axis] = -1 + slice2[axis] = -1 + slice3[axis] = -2 + dx_n = dx[i] if uniform_spacing else dx[i][-1] + # 1D equivalent -- out[-1] = (y[-1] - y[-2]) / (x[-1] - x[-2]) + out[slice1] = (y[slice2] - y[slice3]) / dx_n + + # Numerical differentiation: 2nd order edges + else: + slice1[axis] = 0 + slice2[axis] = 0 + slice3[axis] = 1 + slice4[axis] = 2 + if uniform_spacing: + a = -1.5 / dx[i] + b = 2. / dx[i] + c = -0.5 / dx[i] + else: + dx1 = dx[i][0] + dx2 = dx[i][1] + a = -(2. * dx1 + dx2) / (dx1 * (dx1 + dx2)) + b = (dx1 + dx2) / (dx1 * dx2) + c = - dx1 / (dx2 * (dx1 + dx2)) + # 1D equivalent -- out[0] = a * y[0] + b * y[1] + c * y[2] + out[slice1] = a * y[slice2] + b * y[slice3] + c * y[slice4] + + slice1[axis] = -1 + slice2[axis] = -3 + slice3[axis] = -2 + slice4[axis] = -1 + if uniform_spacing: + a = 0.5 / dx[i] + b = -2. / dx[i] + c = 1.5 / dx[i] + else: + dx1 = dx[i][-2] + dx2 = dx[i][-1] + a = (dx2) / (dx1 * (dx1 + dx2)) + b = - (dx2 + dx1) / (dx1 * dx2) + c = (2. * dx2 + dx1) / (dx2 * (dx1 + dx2)) + # 1D equivalent -- out[-1] = a * f[-3] + b * f[-2] + c * f[-1] + out[slice1] = a * y[slice2] + b * y[slice3] + c * y[slice4] + + outvals.append(out) + + # reset the slice object in this dimension to ":" + slice1[axis] = slice(None) + slice2[axis] = slice(None) + slice3[axis] = slice(None) + slice4[axis] = slice(None) + + if len_axes == 1: + return outvals[0] + else: + return outvals diff --git a/xarray/core/utils.py b/xarray/core/utils.py index c3bb747fac5..9d129d5c4f4 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -591,3 +591,24 @@ def __iter__(self): def __len__(self): num_hidden = sum([k in self._hidden_keys for k in self._data]) return len(self._data) - num_hidden + + +def to_numeric(array, offset=None, datetime_unit=None, dtype=float): + """ + Make datetime array float + + offset: Scalar with the same type of array or None + If None, subtract minimum values to reduce round off error + datetime_unit: None or any of {'Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', + 'us', 'ns', 'ps', 'fs', 'as'} + dtype: target dtype + """ + if array.dtype.kind not in ['m', 'M']: + return array.astype(dtype) + if offset is None: + offset = np.min(array) + array = array - offset + + if datetime_unit: + return (array / np.timedelta64(1, datetime_unit)).astype(dtype) + return array.astype(dtype) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index fc933960914..f8fb9b98ac3 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -15,7 +15,7 @@ from xarray import ( DataArray, Dataset, IndexVariable, MergeError, Variable, align, backends, broadcast, open_dataset, set_options) -from xarray.core import indexing, utils +from xarray.core import indexing, npcompat, utils from xarray.core.common import full_like from xarray.core.pycompat import ( OrderedDict, integer_types, iteritems, unicode_type) @@ -4513,3 +4513,78 @@ def test_raise_no_warning_for_nan_in_binary_ops(): with pytest.warns(None) as record: Dataset(data_vars={'x': ('y', [1, 2, np.NaN])}) > 0 assert len(record) == 0 + + +@pytest.mark.parametrize('dask', [True, False]) +@pytest.mark.parametrize('edge_order', [1, 2]) +def test_gradient(dask, edge_order): + rs = np.random.RandomState(42) + coord = [0.2, 0.35, 0.4, 0.6, 0.7, 0.75, 0.76, 0.8] + + da = xr.DataArray(rs.randn(8, 6), dims=['x', 'y'], + coords={'x': coord, + 'z': 3, 'x2d': (('x', 'y'), rs.randn(8, 6))}) + if dask and has_dask: + da = da.chunk({'x': 4}) + + ds = xr.Dataset({'var': da}) + + # along x + actual = da.differentiate('x', edge_order) + expected_x = xr.DataArray( + npcompat.gradient(da, da['x'], axis=0, edge_order=edge_order), + dims=da.dims, coords=da.coords) + assert_equal(expected_x, actual) + assert_equal(ds['var'].differentiate('x', edge_order=edge_order), + ds.differentiate('x', edge_order=edge_order)['var']) + # coordinate should not change + assert_equal(da['x'], actual['x']) + + # along y + actual = da.differentiate('y', edge_order) + expected_y = xr.DataArray( + npcompat.gradient(da, da['y'], axis=1, edge_order=edge_order), + dims=da.dims, coords=da.coords) + assert_equal(expected_y, actual) + assert_equal(actual, ds.differentiate('y', edge_order=edge_order)['var']) + assert_equal(ds['var'].differentiate('y', edge_order=edge_order), + ds.differentiate('y', edge_order=edge_order)['var']) + + with pytest.raises(ValueError): + da.differentiate('x2d') + + +@pytest.mark.parametrize('dask', [True, False]) +def test_gradient_datetime(dask): + rs = np.random.RandomState(42) + coord = np.array( + ['2004-07-13', '2006-01-13', '2010-08-13', '2010-09-13', + '2010-10-11', '2010-12-13', '2011-02-13', '2012-08-13'], + dtype='datetime64') + + da = xr.DataArray(rs.randn(8, 6), dims=['x', 'y'], + coords={'x': coord, + 'z': 3, 'x2d': (('x', 'y'), rs.randn(8, 6))}) + if dask and has_dask: + da = da.chunk({'x': 4}) + + # along x + actual = da.differentiate('x', edge_order=1, datetime_unit='D') + expected_x = xr.DataArray( + npcompat.gradient( + da, utils.to_numeric(da['x'], datetime_unit='D'), + axis=0, edge_order=1), dims=da.dims, coords=da.coords) + assert_equal(expected_x, actual) + + actual2 = da.differentiate('x', edge_order=1, datetime_unit='h') + assert np.allclose(actual, actual2 * 24) + + # for datetime variable + actual = da['x'].differentiate('x', edge_order=1, datetime_unit='D') + assert np.allclose(actual, 1.0) + + # with different date unit + da = xr.DataArray(coord.astype('datetime64[ms]'), dims=['x'], + coords={'x': coord}) + actual = da.differentiate('x', edge_order=1) + assert np.allclose(actual, 1.0) diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 3f32fc49fd2..b9712f60290 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -12,8 +12,8 @@ from xarray import DataArray, Dataset, concat from xarray.core import duck_array_ops, dtypes from xarray.core.duck_array_ops import ( - array_notnull_equiv, concatenate, count, first, last, mean, rolling_window, - stack, where) + array_notnull_equiv, concatenate, count, first, gradient, last, mean, + rolling_window, stack, where) from xarray.core.pycompat import dask_array_type from xarray.testing import assert_allclose, assert_equal @@ -417,6 +417,23 @@ def test_dask_rolling(axis, window, center): fill_value=np.nan) +@pytest.mark.skipif(not has_dask, reason='This is for dask.') +@pytest.mark.parametrize('axis', [0, -1, 1]) +@pytest.mark.parametrize('edge_order', [1, 2]) +def test_dask_gradient(axis, edge_order): + import dask.array as da + + array = np.array(np.random.randn(100, 5, 40)) + x = np.exp(np.linspace(0, 1, array.shape[axis])) + + darray = da.from_array(array, chunks=[(6, 30, 30, 20, 14), 5, 8]) + expected = gradient(array, x, axis=axis, edge_order=edge_order) + actual = gradient(darray, x, axis=axis, edge_order=edge_order) + + assert isinstance(actual, da.Array) + assert_array_equal(actual, expected) + + @pytest.mark.parametrize('dim_num', [1, 2]) @pytest.mark.parametrize('dtype', [float, int, np.float32, np.bool_]) @pytest.mark.parametrize('dask', [False, True]) From 93f58a60fdb6260f5d5a156c78dca0d956c75fe3 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 21 Sep 2018 18:41:50 -0700 Subject: [PATCH 219/282] Doc fixes for v0.10.9 --- doc/computation.rst | 6 ++++-- doc/roadmap.rst | 2 ++ doc/whats-new.rst | 19 +++++++++++++------ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/doc/computation.rst b/doc/computation.rst index 67cda6f2191..759c87a6cc7 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -208,15 +208,17 @@ coordinates. :py:meth:`~xarray.DataArray.differentiate` computes derivatives by central finite differences using their coordinates, .. ipython:: python - a = xr.DataArray([0, 1, 2, 3], dims=['x'], coords=[0.1, 0.11, 0.2, 0.3]) + + a = xr.DataArray([0, 1, 2, 3], dims=['x'], coords=[[0.1, 0.11, 0.2, 0.3]]) a a.differentiate('x') This method can be used also for multidimensional arrays, .. ipython:: python + a = xr.DataArray(np.arange(8).reshape(4, 2), dims=['x', 'y'], - coords=[0.1, 0.11, 0.2, 0.3]) + coords={'x': [0.1, 0.11, 0.2, 0.3]}) a.differentiate('x') .. note:: diff --git a/doc/roadmap.rst b/doc/roadmap.rst index 2708cb7cf8f..34d203c3f48 100644 --- a/doc/roadmap.rst +++ b/doc/roadmap.rst @@ -1,3 +1,5 @@ +.. _roadmap: + Development roadmap =================== diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 7240059bd10..4a6886159d1 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,11 +27,17 @@ What's New .. _whats-new.0.10.9: -v0.10.9 (unreleased) --------------------- +v0.10.9 (21 September 2019) +--------------------------- -Documentation -~~~~~~~~~~~~~ +This minor release contains a number of backwards compatible enhancements. + +Announcements of note: + +- Xarray is now a NumFOCUS fiscally sponsored project! Read + `the anouncment `_ + for more details. +- We have a new :doc:`roadmap` that outlines our future development plans. Enhancements ~~~~~~~~~~~~ @@ -51,7 +57,8 @@ Enhancements (:issue:`2230`) By `Keisuke Fujii `_. -- :py:meth:`plot()` now accepts the kwargs ``xscale, yscale, xlim, ylim, xticks, yticks`` just like Pandas. Also ``xincrease=False, yincrease=False`` now use matplotlib's axis inverting methods instead of setting limits. +- :py:meth:`plot()` now accepts the kwargs + ``xscale, yscale, xlim, ylim, xticks, yticks`` just like Pandas. Also ``xincrease=False, yincrease=False`` now use matplotlib's axis inverting methods instead of setting limits. By `Deepak Cherian `_. (:issue:`2224`) - DataArray coordinates and Dataset coordinates and data variables are @@ -118,7 +125,7 @@ Bug fixes By `Keisuke Fujii `_. - Now :py:func:`xr.apply_ufunc` raises a ValueError when the size of -``input_core_dims`` is inconsistent with the number of arguments. + ``input_core_dims`` is inconsistent with the number of arguments. (:issue:`2341`) By `Keisuke Fujii `_. From c1ea99212bc3e26789090f6800d468ed7fdd1bb8 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 21 Sep 2018 18:53:20 -0700 Subject: [PATCH 220/282] Revert to dev version for 0.11 --- doc/whats-new.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4a6886159d1..3bc2c521568 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,18 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ +.. _whats-new.0.11.0: + +v0.11.0 (unreleased) +-------------------- + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + + .. _whats-new.0.10.9: v0.10.9 (21 September 2019) From 4577ed891f9839722fd9e606e4f3bdb8e6acef4f Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sat, 22 Sep 2018 13:01:20 +0900 Subject: [PATCH 221/282] misc plotting fixes (#2426) 1. Don't explicitly set rotation for colorbar label. Labels now work better with horizontal colorbar. 2. facetgrid: check if artist is Mappable --- xarray/plot/facetgrid.py | 9 ++++++--- xarray/plot/plot.py | 2 +- xarray/plot/utils.py | 3 ++- xarray/tests/test_plot.py | 13 +++++++++++++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index a0d7c4dd5e2..792b7829bf4 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -502,9 +502,12 @@ def map(self, func, *args, **kwargs): data = self.data.loc[namedict] plt.sca(ax) innerargs = [data[a].values for a in args] - # TODO: is it possible to verify that an artist is mappable? - mappable = func(*innerargs, **kwargs) - self._mappables.append(mappable) + maybe_mappable = func(*innerargs, **kwargs) + # TODO: better way to verify that an artist is mappable? + # https://stackoverflow.com/questions/33023036/is-it-possible-to-detect-if-a-matplotlib-artist-is-a-mappable-suitable-for-use-w#33023522 + if (maybe_mappable and + hasattr(maybe_mappable, 'autoscale_None')): + self._mappables.append(maybe_mappable) self._finalize_grid(*args[:2]) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index b92429b857d..a6add44682f 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -770,7 +770,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, cbar_kwargs.setdefault('cax', cbar_ax) cbar = plt.colorbar(primitive, **cbar_kwargs) if add_labels and 'label' not in cbar_kwargs: - cbar.set_label(label_from_attrs(darray), rotation=90) + cbar.set_label(label_from_attrs(darray)) elif cbar_ax is not None or cbar_kwargs is not None: # inform the user about keywords which aren't used raise ValueError("cbar_ax and cbar_kwargs can't be used with " diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 9af0624dbfc..455d27c3987 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -1,10 +1,11 @@ from __future__ import absolute_import, division, print_function +import textwrap import warnings import numpy as np -import textwrap +from ..core.options import OPTIONS from ..core.pycompat import basestring from ..core.utils import is_scalar from ..core.options import OPTIONS diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index e27f03630b7..b3bc687a5c5 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1051,6 +1051,19 @@ def test_convenient_facetgrid_4d(self): for ax in g.axes.flat: assert ax.has_data() + @pytest.mark.filterwarnings('ignore:This figure includes') + def test_facetgrid_map_only_appends_mappables(self): + a = easy_array((10, 15, 2, 3)) + d = DataArray(a, dims=['y', 'x', 'columns', 'rows']) + g = self.plotfunc(d, x='x', y='y', col='columns', row='rows') + + expected = g._mappables + + g.map(lambda: plt.plot(1, 1)) + actual = g._mappables + + assert expected == actual + def test_facetgrid_cmap(self): # Regression test for GH592 data = (np.random.random(size=(20, 25, 12)) + np.linspace(-3, 3, 12)) From 04253f271c66a12366a82d357c2a889dd3eea42f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 22 Sep 2018 13:13:28 -0700 Subject: [PATCH 222/282] dev/test build for python 3.7 (#2271) * dev/test build for python 3.7 * docs * fixup whatsnew * update 3.7 travis test --- .travis.yml | 2 ++ ci/requirements-py37.yml | 13 +++++++++++++ doc/installing.rst | 2 +- doc/whats-new.rst | 8 +++++--- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 ci/requirements-py37.yml diff --git a/.travis.yml b/.travis.yml index 0e51e946da0..1e6c3254cdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ matrix: env: CONDA_ENV=py35 - python: 3.6 env: CONDA_ENV=py36 + - python: 3.6 # TODO: change this to 3.7 once https://github.com/travis-ci/travis-ci/issues/9815 is fixed + env: CONDA_ENV=py37 - python: 3.6 env: - CONDA_ENV=py36 diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml new file mode 100644 index 00000000000..5f973936f63 --- /dev/null +++ b/ci/requirements-py37.yml @@ -0,0 +1,13 @@ +name: test_env +channels: + - defaults +dependencies: + - python=3.7 + - pip: + - pytest + - flake8 + - mock + - numpy + - pandas + - coveralls + - pytest-cov diff --git a/doc/installing.rst b/doc/installing.rst index 85cd5a02568..eb74eb7162b 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -6,7 +6,7 @@ Installation Required dependencies --------------------- -- Python 2.7 [1]_, 3.5, or 3.6 +- Python 2.7 [1]_, 3.5, 3.6, or 3.7 - `numpy `__ (1.12 or later) - `pandas `__ (0.19.2 or later) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3bc2c521568..67d0d548ec5 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,6 +33,9 @@ v0.11.0 (unreleased) Enhancements ~~~~~~~~~~~~ +- Added support for Python 3.7. (:issue:`2271`). + By `Joe Hamman `_. + Bug fixes ~~~~~~~~~ @@ -78,8 +81,8 @@ Enhancements (:issue:`1186`) By `Seth P `_. - A new CFTimeIndex-enabled :py:func:`cftime_range` function for use in - generating dates from standard or non-standard calendars. By `Spencer Clark - `_. + generating dates from standard or non-standard calendars. By `Spencer Clark + `_. - When interpolating over a ``datetime64`` axis, you can now provide a datetime string instead of a ``datetime64`` object. E.g. ``da.interp(time='1991-02-01')`` (:issue:`2284`) @@ -178,7 +181,6 @@ Enhancements :py:meth:`~xarray.DataArray.from_cdms2` (:issue:`2262`). By `Stephane Raynaud `_. - Bug fixes ~~~~~~~~~ From 1ec83a75c409c68683ac035dfee1c26f8cbc6695 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 26 Sep 2018 09:55:32 +0900 Subject: [PATCH 223/282] make facetgrid execute _finalize() only once (#2435) * only apply finalize_grid once. * Add test --- xarray/plot/facetgrid.py | 16 ++++++++++------ xarray/tests/test_plot.py | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index 792b7829bf4..79a3993e23b 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -188,6 +188,7 @@ def __init__(self, data, col=None, row=None, col_wrap=None, self._y_var = None self._cmap_extend = None self._mappables = [] + self._finalized = False @property def _left_axes(self): @@ -308,13 +309,16 @@ def map_dataarray_line(self, x=None, y=None, hue=None, **kwargs): def _finalize_grid(self, *axlabels): """Finalize the annotations and layout.""" - self.set_axis_labels(*axlabels) - self.set_titles() - self.fig.tight_layout() + if not self._finalized: + self.set_axis_labels(*axlabels) + self.set_titles() + self.fig.tight_layout() - for ax, namedict in zip(self.axes.flat, self.name_dicts.flat): - if namedict is None: - ax.set_visible(False) + for ax, namedict in zip(self.axes.flat, self.name_dicts.flat): + if namedict is None: + ax.set_visible(False) + + self._finalized = True def add_legend(self, **kwargs): figlegend = self.fig.legend( diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index b3bc687a5c5..1423f7ae853 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1523,7 +1523,9 @@ def test_num_ticks(self): @pytest.mark.slow def test_map(self): + assert self.g._finalized is False self.g.map(plt.contourf, 'x', 'y', Ellipsis) + assert self.g._finalized is True self.g.map(lambda: None) @pytest.mark.slow From 1857a7fc2ab3a472025ff5d69371feaf7e3c4d74 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 26 Sep 2018 16:27:54 -0700 Subject: [PATCH 224/282] switch travis language to generic (#2432) * dev/test build for python 3.7 * docs * fixup whatsnew * update 3.7 travis test * generic travis config * switch to minimal --- .travis.yml | 80 +++++++++++++++++++---------------------------------- 1 file changed, 28 insertions(+), 52 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e6c3254cdd..defb37ec8aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ # Based on http://conda.pydata.org/docs/travis.html -language: python +language: minimal sudo: false # use container based build notifications: email: false @@ -10,74 +10,48 @@ branches: matrix: fast_finish: true include: - - python: 2.7 - env: CONDA_ENV=py27-min - - python: 2.7 - env: CONDA_ENV=py27-cdat+iris+pynio - - python: 3.5 - env: CONDA_ENV=py35 - - python: 3.6 - env: CONDA_ENV=py36 - - python: 3.6 # TODO: change this to 3.7 once https://github.com/travis-ci/travis-ci/issues/9815 is fixed - env: CONDA_ENV=py37 - - python: 3.6 - env: + - env: CONDA_ENV=py27-min + - env: CONDA_ENV=py27-cdat+iris+pynio + - env: CONDA_ENV=py35 + - env: CONDA_ENV=py36 + - env: CONDA_ENV=py37 + - env: - CONDA_ENV=py36 - EXTRA_FLAGS="--run-flaky --run-network-tests" - - python: 3.6 - env: CONDA_ENV=py36-netcdf4-dev + - env: CONDA_ENV=py36-netcdf4-dev addons: apt_packages: - libhdf5-serial-dev - netcdf-bin - libnetcdf-dev - - python: 3.6 - env: CONDA_ENV=py36-dask-dev - - python: 3.6 - env: CONDA_ENV=py36-pandas-dev - - python: 3.6 - env: CONDA_ENV=py36-bottleneck-dev - - python: 3.6 - env: CONDA_ENV=py36-condaforge-rc - - python: 3.6 - env: CONDA_ENV=py36-pynio-dev - - python: 3.6 - env: CONDA_ENV=py36-rasterio-0.36 - - python: 3.6 - env: CONDA_ENV=py36-zarr-dev - - python: 3.5 - env: CONDA_ENV=docs - - python: 3.6 - env: CONDA_ENV=py36-hypothesis + - env: CONDA_ENV=py36-dask-dev + - env: CONDA_ENV=py36-pandas-dev + - env: CONDA_ENV=py36-bottleneck-dev + - env: CONDA_ENV=py36-condaforge-rc + - env: CONDA_ENV=py36-pynio-dev + - env: CONDA_ENV=py36-rasterio-0.36 + - env: CONDA_ENV=py36-zarr-dev + - env: CONDA_ENV=docs + - env: CONDA_ENV=py36-hypothesis + allow_failures: - - python: 3.6 - env: + - env: - CONDA_ENV=py36 - EXTRA_FLAGS="--run-flaky --run-network-tests" - - python: 3.6 - env: CONDA_ENV=py36-netcdf4-dev + - env: CONDA_ENV=py36-netcdf4-dev addons: apt_packages: - libhdf5-serial-dev - netcdf-bin - libnetcdf-dev - - python: 3.6 - env: CONDA_ENV=py36-pandas-dev - - python: 3.6 - env: CONDA_ENV=py36-bottleneck-dev - - python: 3.6 - env: CONDA_ENV=py36-condaforge-rc - - python: 3.6 - env: CONDA_ENV=py36-pynio-dev - - python: 3.6 - env: CONDA_ENV=py36-zarr-dev + - env: CONDA_ENV=py36-pandas-dev + - env: CONDA_ENV=py36-bottleneck-dev + - env: CONDA_ENV=py36-condaforge-rc + - env: CONDA_ENV=py36-pynio-dev + - env: CONDA_ENV=py36-zarr-dev before_install: - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget http://repo.continuum.io/miniconda/Miniconda-3.16.0-Linux-x86_64.sh -O miniconda.sh; - else - wget http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh -O miniconda.sh; - fi + - wget http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - hash -r @@ -97,6 +71,8 @@ install: - python xarray/util/print_versions.py script: + - which python + - python --version - python -OO -c "import xarray" - if [[ "$CONDA_ENV" == "docs" ]]; then conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc; From 96dde664eda26a76f934151dd10dc02f6cb0000b Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Thu, 27 Sep 2018 09:47:27 +1000 Subject: [PATCH 225/282] Use profile mechanism, not no-op mutation (#2442) --- properties/test_encode_decode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/properties/test_encode_decode.py b/properties/test_encode_decode.py index 8d84c0f6815..7b3e75fbf0c 100644 --- a/properties/test_encode_decode.py +++ b/properties/test_encode_decode.py @@ -13,7 +13,8 @@ import xarray as xr # Run for a while - arrays are a bigger search space than usual -settings.deadline = None +settings.register_profile("ci", deadline=None) +settings.load_profile("ci") an_array = npst.arrays( From 78058e2c1f39cbfae6eddb30e3b7d4a81b54ad8b Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 27 Sep 2018 18:37:02 +0200 Subject: [PATCH 226/282] Remove incorrect statement about "drop" in the text docs (#2439) As pointed out by FaustinCarter in GH1949 --- doc/data-structures.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/doc/data-structures.rst b/doc/data-structures.rst index 10d83ca448f..618ccccff3e 100644 --- a/doc/data-structures.rst +++ b/doc/data-structures.rst @@ -408,13 +408,6 @@ operations keep around coordinates: list(ds[['x']]) list(ds.drop('temperature')) -If a dimension name is given as an argument to ``drop``, it also drops all -variables that use that dimension: - -.. ipython:: python - - list(ds.drop('time')) - As an alternate to dictionary-like modifications, you can use :py:meth:`~xarray.Dataset.assign` and :py:meth:`~xarray.Dataset.assign_coords`. These methods return a new dataset with additional (or replaced) or values: From 638b251c622359b665208276a2cb23b0fbc5141b Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 28 Sep 2018 08:54:29 +0200 Subject: [PATCH 227/282] Future warning for default reduction dimension of groupby (#2366) * warn the default reduction dimension of groupby * Only use DEFAULT_DIMS in groupby/resample * Restore unintended line break * Lint * Add whatsnew * Support dataset.groupby * Add a version in the warning message. * is -> == * Update tests * Update docs for DatasetResample.reduce * Match dataset.resample behavior to the current one. * Update via comments. --- doc/whats-new.rst | 10 ++++++ xarray/__init__.py | 2 ++ xarray/core/common.py | 6 +++- xarray/core/dataset.py | 11 +++--- xarray/core/groupby.py | 64 ++++++++++++++++++++++++++++++++-- xarray/core/resample.py | 12 ++++--- xarray/core/variable.py | 2 ++ xarray/tests/test_dask.py | 4 +-- xarray/tests/test_dataarray.py | 36 +++++++++++++++---- xarray/tests/test_dataset.py | 23 +++++++----- 10 files changed, 141 insertions(+), 29 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 67d0d548ec5..4e1607f0e42 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -30,6 +30,16 @@ What's New v0.11.0 (unreleased) -------------------- +Breaking changes +~~~~~~~~~~~~~~~~ + +- Reduction of :py:meth:`DataArray.groupby` and :py:meth:`DataArray.resample` + without dimension argument will change in the next release. + Now we warn a FutureWarning. + By `Keisuke Fujii `_. + +Documentation +~~~~~~~~~~~~~ Enhancements ~~~~~~~~~~~~ diff --git a/xarray/__init__.py b/xarray/__init__.py index e3898f348cc..59a961c6b56 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -34,3 +34,5 @@ from . import tutorial from . import ufuncs from . import testing + +from .core.common import ALL_DIMS diff --git a/xarray/core/common.py b/xarray/core/common.py index 280034a30dd..41e4fec2982 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -10,7 +10,11 @@ from . import duck_array_ops, dtypes, formatting, ops from .arithmetic import SupportsArithmetic from .pycompat import OrderedDict, basestring, dask_array_type, suppress -from .utils import either_dict_or_kwargs, Frozen, SortedKeysDict +from .utils import either_dict_or_kwargs, Frozen, SortedKeysDict, ReprObject + + +# Used as a sentinel value to indicate a all dimensions +ALL_DIMS = ReprObject('') class ImplementsArrayReduce(object): diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 9cf304858a6..981ad3157ba 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -18,7 +18,8 @@ from .. import conventions from .alignment import align from .common import ( - DataWithCoords, ImplementsDatasetReduce, _contains_datetime_like_objects) + ALL_DIMS, DataWithCoords, ImplementsDatasetReduce, + _contains_datetime_like_objects) from .coordinates import ( DatasetCoordinates, Indexes, LevelCoordinatesSource, assert_coordinate_consistent, remap_label_indexers) @@ -743,7 +744,7 @@ def copy(self, deep=False, data=None): Shallow copy versus deep copy >>> da = xr.DataArray(np.random.randn(2, 3)) - >>> ds = xr.Dataset({'foo': da, 'bar': ('x', [-1, 2])}, + >>> ds = xr.Dataset({'foo': da, 'bar': ('x', [-1, 2])}, coords={'x': ['one', 'two']}) >>> ds.copy() @@ -775,7 +776,7 @@ def copy(self, deep=False, data=None): foo (dim_0, dim_1) float64 7.0 0.3897 -1.862 -0.6091 -1.051 -0.3003 bar (x) int64 -1 2 - Changing the data using the ``data`` argument maintains the + Changing the data using the ``data`` argument maintains the structure of the original object, but with the new data. Original object is unaffected. @@ -826,7 +827,7 @@ def copy(self, deep=False, data=None): # skip __init__ to avoid costly validation return self._construct_direct(variables, self._coord_names.copy(), self._dims.copy(), self._attrs_copy(), - encoding=self.encoding) + encoding=self.encoding) def _subset_with_all_valid_coords(self, variables, coord_names, attrs): needed_dims = set() @@ -2893,6 +2894,8 @@ def reduce(self, func, dim=None, keep_attrs=False, numeric_only=False, Dataset with this object's DataArrays replaced with new DataArrays of summarized data and the indicated dimension(s) removed. """ + if dim is ALL_DIMS: + dim = None if isinstance(dim, basestring): dims = set([dim]) elif dim is None: diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index 7068f8e6cae..3842c642047 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -1,14 +1,15 @@ from __future__ import absolute_import, division, print_function import functools +import warnings import numpy as np import pandas as pd -from . import dtypes, duck_array_ops, nputils, ops +from . import dtypes, duck_array_ops, nputils, ops, utils from .arithmetic import SupportsArithmetic from .combine import concat -from .common import ImplementsArrayReduce, ImplementsDatasetReduce +from .common import ALL_DIMS, ImplementsArrayReduce, ImplementsDatasetReduce from .pycompat import integer_types, range, zip from .utils import hashable, maybe_wrap_array, peek_at, safe_cast_to_index from .variable import IndexVariable, Variable, as_variable @@ -567,10 +568,39 @@ def reduce(self, func, dim=None, axis=None, keep_attrs=False, Array with summarized data and the indicated dimension(s) removed. """ + if dim == DEFAULT_DIMS: + dim = ALL_DIMS + # TODO change this to dim = self._group_dim after + # the deprecation process + if self._obj.ndim > 1: + warnings.warn( + "Default reduction dimension will be changed to the " + "grouped dimension after xarray 0.12. To silence this " + "warning, pass dim=xarray.ALL_DIMS explicitly.", + FutureWarning, stacklevel=2) + def reduce_array(ar): return ar.reduce(func, dim, axis, keep_attrs=keep_attrs, **kwargs) return self.apply(reduce_array, shortcut=shortcut) + # TODO remove the following class method and DEFAULT_DIMS after the + # deprecation cycle + @classmethod + def _reduce_method(cls, func, include_skipna, numeric_only): + if include_skipna: + def wrapped_func(self, dim=DEFAULT_DIMS, axis=None, skipna=None, + keep_attrs=False, **kwargs): + return self.reduce(func, dim, axis, keep_attrs=keep_attrs, + skipna=skipna, allow_lazy=True, **kwargs) + else: + def wrapped_func(self, dim=DEFAULT_DIMS, axis=None, + keep_attrs=False, **kwargs): + return self.reduce(func, dim, axis, keep_attrs=keep_attrs, + allow_lazy=True, **kwargs) + return wrapped_func + + +DEFAULT_DIMS = utils.ReprObject('') ops.inject_reduce_methods(DataArrayGroupBy) ops.inject_binary_ops(DataArrayGroupBy) @@ -649,10 +679,40 @@ def reduce(self, func, dim=None, keep_attrs=False, **kwargs): Array with summarized data and the indicated dimension(s) removed. """ + if dim == DEFAULT_DIMS: + dim = ALL_DIMS + # TODO change this to dim = self._group_dim after + # the deprecation process. Do not forget to remove _reduce_method + warnings.warn( + "Default reduction dimension will be changed to the " + "grouped dimension after xarray 0.12. To silence this " + "warning, pass dim=xarray.ALL_DIMS explicitly.", + FutureWarning, stacklevel=2) + elif dim is None: + dim = self._group_dim + def reduce_dataset(ds): return ds.reduce(func, dim, keep_attrs, **kwargs) return self.apply(reduce_dataset) + # TODO remove the following class method and DEFAULT_DIMS after the + # deprecation cycle + @classmethod + def _reduce_method(cls, func, include_skipna, numeric_only): + if include_skipna: + def wrapped_func(self, dim=DEFAULT_DIMS, keep_attrs=False, + skipna=None, **kwargs): + return self.reduce(func, dim, keep_attrs, skipna=skipna, + numeric_only=numeric_only, allow_lazy=True, + **kwargs) + else: + def wrapped_func(self, dim=DEFAULT_DIMS, keep_attrs=False, + **kwargs): + return self.reduce(func, dim, keep_attrs, + numeric_only=numeric_only, allow_lazy=True, + **kwargs) + return wrapped_func + def assign(self, **kwargs): """Assign data variables by group. diff --git a/xarray/core/resample.py b/xarray/core/resample.py index 4933a09b257..25c149c51af 100644 --- a/xarray/core/resample.py +++ b/xarray/core/resample.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function from . import ops -from .groupby import DataArrayGroupBy, DatasetGroupBy +from .groupby import DataArrayGroupBy, DatasetGroupBy, DEFAULT_DIMS from .pycompat import OrderedDict, dask_array_type RESAMPLE_DIM = '__resample_dim__' @@ -277,15 +277,14 @@ def reduce(self, func, dim=None, keep_attrs=False, **kwargs): """Reduce the items in this group by applying `func` along the pre-defined resampling dimension. - Note that `dim` is by default here and ignored if passed by the user; - this ensures compatibility with the existing reduce interface. - Parameters ---------- func : function Function which can be called in the form `func(x, axis=axis, **kwargs)` to return the result of collapsing an np.ndarray over an integer valued axis. + dim : str or sequence of str, optional + Dimension(s) over which to apply `func`. keep_attrs : bool, optional If True, the datasets's attributes (`attrs`) will be copied from the original object to the new one. If False (default), the new @@ -299,8 +298,11 @@ def reduce(self, func, dim=None, keep_attrs=False, **kwargs): Array with summarized data and the indicated dimension(s) removed. """ + if dim == DEFAULT_DIMS: + dim = None + return super(DatasetResample, self).reduce( - func, self._dim, keep_attrs, **kwargs) + func, dim, keep_attrs, **kwargs) def _interpolate(self, kind='linear'): """Apply scipy.interpolate.interp1d along resampling dimension.""" diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 86629cc2a28..c003d52aab2 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1333,6 +1333,8 @@ def reduce(self, func, dim=None, axis=None, keep_attrs=False, Array with summarized data and the indicated dimension(s) removed. """ + if dim is common.ALL_DIMS: + dim = None if dim is not None and axis is not None: raise ValueError("cannot supply both 'axis' and 'dim' arguments") diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 6ca83ab73ab..43fa35473ce 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -385,8 +385,8 @@ def test_groupby(self): u = self.eager_array v = self.lazy_array - expected = u.groupby('x').mean() - actual = v.groupby('x').mean() + expected = u.groupby('x').mean(xr.ALL_DIMS) + actual = v.groupby('x').mean(xr.ALL_DIMS) self.assertLazyAndAllClose(expected, actual) def test_groupby_first(self): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 2b93e696d50..f8b288f4ab0 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2,6 +2,7 @@ import pickle from copy import deepcopy +from distutils.version import LooseVersion from textwrap import dedent import warnings @@ -14,7 +15,7 @@ DataArray, Dataset, IndexVariable, Variable, align, broadcast, set_options) from xarray.convert import from_cdms2 from xarray.coding.times import CFDatetimeCoder, _import_cftime -from xarray.core.common import full_like +from xarray.core.common import full_like, ALL_DIMS from xarray.core.pycompat import OrderedDict, iteritems from xarray.tests import ( ReturnItem, TestCase, assert_allclose, assert_array_equal, assert_equal, @@ -2000,15 +2001,15 @@ def test_groupby_sum(self): self.x[:, 10:].sum(), self.x[:, 9:10].sum()]).T), 'abc': Variable(['abc'], np.array(['a', 'b', 'c']))})['foo'] - assert_allclose(expected_sum_all, grouped.reduce(np.sum)) - assert_allclose(expected_sum_all, grouped.sum()) + assert_allclose(expected_sum_all, grouped.reduce(np.sum, dim=ALL_DIMS)) + assert_allclose(expected_sum_all, grouped.sum(ALL_DIMS)) expected = DataArray([array['y'].values[idx].sum() for idx in [slice(9), slice(10, None), slice(9, 10)]], [['a', 'b', 'c']], ['abc']) actual = array['y'].groupby('abc').apply(np.sum) assert_allclose(expected, actual) - actual = array['y'].groupby('abc').sum() + actual = array['y'].groupby('abc').sum(ALL_DIMS) assert_allclose(expected, actual) expected_sum_axis1 = Dataset( @@ -2019,6 +2020,27 @@ def test_groupby_sum(self): assert_allclose(expected_sum_axis1, grouped.reduce(np.sum, 'y')) assert_allclose(expected_sum_axis1, grouped.sum('y')) + def test_groupby_warning(self): + array = self.make_groupby_example_array() + grouped = array.groupby('y') + with pytest.warns(FutureWarning): + grouped.sum() + + @pytest.mark.skipif(LooseVersion(xr.__version__) < LooseVersion('0.12'), + reason="not to forget the behavior change") + def test_groupby_sum_default(self): + array = self.make_groupby_example_array() + grouped = array.groupby('abc') + + expected_sum_all = Dataset( + {'foo': Variable(['x', 'abc'], + np.array([self.x[:, :9].sum(axis=-1), + self.x[:, 10:].sum(axis=-1), + self.x[:, 9:10].sum(axis=-1)]).T), + 'abc': Variable(['abc'], np.array(['a', 'b', 'c']))})['foo'] + + assert_allclose(expected_sum_all, grouped.sum()) + def test_groupby_count(self): array = DataArray( [0, 0, np.nan, np.nan, 0, 0], @@ -2099,9 +2121,9 @@ def test_groupby_math(self): assert_identical(expected, actual) grouped = array.groupby('abc') - expected_agg = (grouped.mean() - np.arange(3)).rename(None) + expected_agg = (grouped.mean(ALL_DIMS) - np.arange(3)).rename(None) actual = grouped - DataArray(range(3), [('abc', ['a', 'b', 'c'])]) - actual_agg = actual.groupby('abc').mean() + actual_agg = actual.groupby('abc').mean(ALL_DIMS) assert_allclose(expected_agg, actual_agg) with raises_regex(TypeError, 'only support binary ops'): @@ -2175,7 +2197,7 @@ def test_groupby_multidim(self): ('lon', DataArray([5, 28, 23], coords=[('lon', [30., 40., 50.])])), ('lat', DataArray([16, 40], coords=[('lat', [10., 20.])]))]: - actual_sum = array.groupby(dim).sum() + actual_sum = array.groupby(dim).sum(ALL_DIMS) assert_identical(expected_sum, actual_sum) def test_groupby_multidim_apply(self): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index f8fb9b98ac3..237dc09d06a 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -14,7 +14,7 @@ import xarray as xr from xarray import ( DataArray, Dataset, IndexVariable, MergeError, Variable, align, backends, - broadcast, open_dataset, set_options) + broadcast, open_dataset, set_options, ALL_DIMS) from xarray.core import indexing, npcompat, utils from xarray.core.common import full_like from xarray.core.pycompat import ( @@ -2648,20 +2648,28 @@ def test_groupby_reduce(self): expected = data.mean('y') expected['yonly'] = expected['yonly'].variable.set_dims({'x': 3}) - actual = data.groupby('x').mean() + actual = data.groupby('x').mean(ALL_DIMS) assert_allclose(expected, actual) actual = data.groupby('x').mean('y') assert_allclose(expected, actual) letters = data['letters'] - expected = Dataset({'xy': data['xy'].groupby(letters).mean(), + expected = Dataset({'xy': data['xy'].groupby(letters).mean(ALL_DIMS), 'xonly': (data['xonly'].mean().variable .set_dims({'letters': 2})), 'yonly': data['yonly'].groupby(letters).mean()}) - actual = data.groupby('letters').mean() + actual = data.groupby('letters').mean(ALL_DIMS) assert_allclose(expected, actual) + def test_groupby_warn(self): + data = Dataset({'xy': (['x', 'y'], np.random.randn(3, 4)), + 'xonly': ('x', np.random.randn(3)), + 'yonly': ('y', np.random.randn(4)), + 'letters': ('y', ['a', 'a', 'b', 'b'])}) + with pytest.warns(FutureWarning): + data.groupby('x').mean() + def test_groupby_math(self): def reorder_dims(x): return x.transpose('dim1', 'dim2', 'dim3', 'time') @@ -2716,7 +2724,7 @@ def test_groupby_math_virtual(self): ds = Dataset({'x': ('t', [1, 2, 3])}, {'t': pd.date_range('20100101', periods=3)}) grouped = ds.groupby('t.day') - actual = grouped - grouped.mean() + actual = grouped - grouped.mean(ALL_DIMS) expected = Dataset({'x': ('t', [0, 0, 0])}, ds[['t', 't.day']]) assert_identical(actual, expected) @@ -2725,18 +2733,17 @@ def test_groupby_nan(self): # nan should be excluded from groupby ds = Dataset({'foo': ('x', [1, 2, 3, 4])}, {'bar': ('x', [1, 1, 2, np.nan])}) - actual = ds.groupby('bar').mean() + actual = ds.groupby('bar').mean(ALL_DIMS) expected = Dataset({'foo': ('bar', [1.5, 3]), 'bar': [1, 2]}) assert_identical(actual, expected) def test_groupby_order(self): # groupby should preserve variables order - ds = Dataset() for vn in ['a', 'b', 'c']: ds[vn] = DataArray(np.arange(10), dims=['t']) data_vars_ref = list(ds.data_vars.keys()) - ds = ds.groupby('t').mean() + ds = ds.groupby('t').mean(ALL_DIMS) data_vars = list(ds.data_vars.keys()) assert data_vars == data_vars_ref # coords are now at the end of the list, so the test below fails From 458cf51ce20e8d924b38b59c8fbc3bb10f39148e Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 28 Sep 2018 15:44:28 +0200 Subject: [PATCH 228/282] restore ddof support in std (#2447) * restore ddof support in std * whats new --- doc/whats-new.rst | 3 +++ xarray/core/nanops.py | 4 ++-- xarray/tests/test_duck_array_ops.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4e1607f0e42..40d21cc5346 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -49,6 +49,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- ``xarray.DataArray.std()`` now correctly accepts ``ddof`` keyword argument. + (:issue:`2240`) + By `Keisuke Fujii `_. .. _whats-new.0.10.9: diff --git a/xarray/core/nanops.py b/xarray/core/nanops.py index 2309ed9619d..9549c8e77b9 100644 --- a/xarray/core/nanops.py +++ b/xarray/core/nanops.py @@ -184,9 +184,9 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0): a, axis=axis, dtype=dtype, ddof=ddof) -def nanstd(a, axis=None, dtype=None, out=None): +def nanstd(a, axis=None, dtype=None, out=None, ddof=0): return _dask_or_eager_func('nanstd', eager_module=nputils)( - a, axis=axis, dtype=dtype) + a, axis=axis, dtype=dtype, ddof=ddof) def nanprod(a, axis=None, dtype=None, out=None, min_count=None): diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index b9712f60290..aab5d305a82 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -309,7 +309,7 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): assert_allclose(actual, expected, rtol=rtol) # make sure the compatiblility with pandas' results. - if func == 'var': + if func in ['var', 'std']: expected = series_reduce(da, func, skipna=skipna, dim=aggdim, ddof=0) assert_allclose(actual, expected, rtol=rtol) From c2b09d697c741b5d6ddede0ba01076c0cb09cf19 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 28 Sep 2018 09:44:54 -0400 Subject: [PATCH 229/282] Enable use of cftime.datetime coordinates with differentiate and interp (#2434) * Enable use of cftime.datetime coords with differentiate and interp * Raise TypeError for non-datetime x_new * Rename to_numeric to datetime_to_numeric --- doc/interpolation.rst | 3 ++ doc/time-series.rst | 56 ++++++++++++++------- doc/whats-new.rst | 7 +++ xarray/coding/cftimeindex.py | 29 +++++++++++ xarray/core/dataset.py | 51 +++++++++++++------ xarray/core/missing.py | 12 +++-- xarray/core/utils.py | 17 ++++--- xarray/tests/test_cftimeindex.py | 23 ++++++++- xarray/tests/test_dataset.py | 40 +++++++++++++-- xarray/tests/test_interp.py | 85 +++++++++++++++++++++++++++++++- xarray/tests/test_utils.py | 43 +++++++++++++++- 11 files changed, 312 insertions(+), 54 deletions(-) diff --git a/doc/interpolation.rst b/doc/interpolation.rst index e5230e95dae..10e46331d0a 100644 --- a/doc/interpolation.rst +++ b/doc/interpolation.rst @@ -63,6 +63,9 @@ by specifing the time periods required. da_dt64.interp(time=pd.date_range('1/1/2000', '1/3/2000', periods=3)) +Interpolation of data indexed by a :py:class:`~xarray.CFTimeIndex` is also +allowed. See :ref:`CFTimeIndex` for examples. + .. note:: Currently, our interpolation only works for regular grids. diff --git a/doc/time-series.rst b/doc/time-series.rst index d99c3218d18..c1a686b409f 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -70,9 +70,9 @@ You can manual decode arrays in this form by passing a dataset to One unfortunate limitation of using ``datetime64[ns]`` is that it limits the native representation of dates to those that fall between the years 1678 and 2262. When a netCDF file contains dates outside of these bounds, dates will be -returned as arrays of ``cftime.datetime`` objects and a ``CFTimeIndex`` -can be used for indexing. The ``CFTimeIndex`` enables only a subset of -the indexing functionality of a ``pandas.DatetimeIndex`` and is only enabled +returned as arrays of :py:class:`cftime.datetime` objects and a :py:class:`~xarray.CFTimeIndex` +can be used for indexing. The :py:class:`~xarray.CFTimeIndex` enables only a subset of +the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only enabled when using the standalone version of ``cftime`` (not the version packaged with earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more information. @@ -219,12 +219,12 @@ Non-standard calendars and dates outside the Timestamp-valid range ------------------------------------------------------------------ Through the standalone ``cftime`` library and a custom subclass of -``pandas.Index``, xarray supports a subset of the indexing functionality enabled -through the standard ``pandas.DatetimeIndex`` for dates from non-standard -calendars or dates using a standard calendar, but outside the -`Timestamp-valid range`_ (approximately between years 1678 and 2262). This -behavior has not yet been turned on by default; to take advantage of this -functionality, you must have the ``enable_cftimeindex`` option set to +:py:class:`pandas.Index`, xarray supports a subset of the indexing +functionality enabled through the standard :py:class:`pandas.DatetimeIndex` for +dates from non-standard calendars or dates using a standard calendar, but +outside the `Timestamp-valid range`_ (approximately between years 1678 and +2262). This behavior has not yet been turned on by default; to take advantage +of this functionality, you must have the ``enable_cftimeindex`` option set to ``True`` within your context (see :py:func:`~xarray.set_options` for more information). It is expected that this will become the default behavior in xarray version 0.11. @@ -232,7 +232,7 @@ xarray version 0.11. For instance, you can create a DataArray indexed by a time coordinate with a no-leap calendar within a context manager setting the ``enable_cftimeindex`` option, and the time index will be cast to a -``CFTimeIndex``: +:py:class:`~xarray.CFTimeIndex`: .. ipython:: python @@ -247,28 +247,28 @@ coordinate with a no-leap calendar within a context manager setting the .. note:: - With the ``enable_cftimeindex`` option activated, a ``CFTimeIndex`` + With the ``enable_cftimeindex`` option activated, a :py:class:`~xarray.CFTimeIndex` will be used for time indexing if any of the following are true: - The dates are from a non-standard calendar - Any dates are outside the Timestamp-valid range - Otherwise a ``pandas.DatetimeIndex`` will be used. In addition, if any + Otherwise a :py:class:`pandas.DatetimeIndex` will be used. In addition, if any variable (not just an index variable) is encoded using a non-standard - calendar, its times will be decoded into ``cftime.datetime`` objects, + calendar, its times will be decoded into :py:class:`cftime.datetime` objects, regardless of whether or not they can be represented using ``np.datetime64[ns]`` objects. -xarray also includes a :py:func:`cftime_range` function, which enables creating a -``CFTimeIndex`` with regularly-spaced dates. For instance, we can create the -same dates and DataArray we created above using: +xarray also includes a :py:func:`~xarray.cftime_range` function, which enables +creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates. For instance, we can +create the same dates and DataArray we created above using: .. ipython:: python dates = xr.cftime_range(start='0001', periods=24, freq='MS', calendar='noleap') da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], name='foo') -For data indexed by a ``CFTimeIndex`` xarray currently supports: +For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: - `Partial datetime string indexing`_ using strictly `ISO 8601-format`_ partial datetime strings: @@ -294,7 +294,25 @@ For data indexed by a ``CFTimeIndex`` xarray currently supports: .. ipython:: python da.groupby('time.month').sum() - + +- Interpolation using :py:class:`cftime.datetime` objects: + +.. ipython:: python + + da.interp(time=[DatetimeNoLeap(1, 1, 15), DatetimeNoLeap(1, 2, 15)]) + +- Interpolation using datetime strings: + +.. ipython:: python + + da.interp(time=['0001-01-15', '0001-02-15']) + +- Differentiation: + +.. ipython:: python + + da.differentiate('time') + - And serialization: .. ipython:: python @@ -305,7 +323,7 @@ For data indexed by a ``CFTimeIndex`` xarray currently supports: .. note:: Currently resampling along the time dimension for data indexed by a - ``CFTimeIndex`` is not supported. + :py:class:`~xarray.CFTimeIndex` is not supported. .. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timestamp-limitations .. _ISO 8601-format: https://en.wikipedia.org/wiki/ISO_8601 diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 40d21cc5346..a5b7b36142e 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -46,6 +46,13 @@ Enhancements - Added support for Python 3.7. (:issue:`2271`). By `Joe Hamman `_. +- Added support for using ``cftime.datetime`` coordinates with + :py:meth:`~xarray.DataArray.differentiate`, + :py:meth:`~xarray.Dataset.differentiate`, + :py:meth:`~xarray.DataArray.interp`, and + :py:meth:`~xarray.Dataset.interp`. + By `Spencer Clark `_ + Bug fixes ~~~~~~~~~ diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index ea2bcbc5858..e236dca3693 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -314,3 +314,32 @@ def __contains__(self, key): def contains(self, key): """Needed for .loc based partial-string indexing""" return self.__contains__(key) + + +def _parse_iso8601_without_reso(date_type, datetime_str): + date, _ = _parse_iso8601_with_reso(date_type, datetime_str) + return date + + +def _parse_array_of_cftime_strings(strings, date_type): + """Create a numpy array from an array of strings. + + For use in generating dates from strings for use with interp. Assumes the + array is either 0-dimensional or 1-dimensional. + + Parameters + ---------- + strings : array of strings + Strings to convert to dates + date_type : cftime.datetime type + Calendar type to use for dates + + Returns + ------- + np.array + """ + if strings.ndim == 0: + return np.array(_parse_iso8601_without_reso(date_type, strings.item())) + else: + return np.array([_parse_iso8601_without_reso(date_type, s) + for s in strings]) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 981ad3157ba..4ad5902ebad 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -32,9 +32,11 @@ OrderedDict, basestring, dask_array_type, integer_types, iteritems, range) from .utils import ( Frozen, SortedKeysDict, either_dict_or_kwargs, decode_numpy_dict_values, - ensure_us_time_resolution, hashable, maybe_wrap_array, to_numeric) + ensure_us_time_resolution, hashable, maybe_wrap_array, datetime_to_numeric) from .variable import IndexVariable, Variable, as_variable, broadcast_variables +from ..coding.cftimeindex import _parse_array_of_cftime_strings + # list of attributes of pd.DatetimeIndex that are ndarrays of time info _DATETIMEINDEX_COMPONENTS = ['year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond', 'nanosecond', 'date', @@ -1413,8 +1415,8 @@ def _validate_indexers(self, indexers): """ Here we make sure + indexer has a valid keys + indexer is in a valid data type - * string indexers are cast to datetime64 - if associated index is DatetimeIndex + + string indexers are cast to the appropriate date type if the + associated index is a DatetimeIndex or CFTimeIndex """ from .dataarray import DataArray @@ -1436,10 +1438,12 @@ def _validate_indexers(self, indexers): else: v = np.asarray(v) - if ((v.dtype.kind == 'U' or v.dtype.kind == 'S') - and isinstance(self.coords[k].to_index(), - pd.DatetimeIndex)): - v = v.astype('datetime64[ns]') + if v.dtype.kind == 'U' or v.dtype.kind == 'S': + index = self.indexes[k] + if isinstance(index, pd.DatetimeIndex): + v = v.astype('datetime64[ns]') + elif isinstance(index, xr.CFTimeIndex): + v = _parse_array_of_cftime_strings(v, index.date_type) if v.ndim == 0: v = as_variable(v) @@ -1981,11 +1985,26 @@ def maybe_variable(obj, k): except KeyError: return as_variable((k, range(obj.dims[k]))) + def _validate_interp_indexer(x, new_x): + # In the case of datetimes, the restrictions placed on indexers + # used with interp are stronger than those which are placed on + # isel, so we need an additional check after _validate_indexers. + if (_contains_datetime_like_objects(x) and + not _contains_datetime_like_objects(new_x)): + raise TypeError('When interpolating over a datetime-like ' + 'coordinate, the coordinates to ' + 'interpolate to must be either datetime ' + 'strings or datetimes. ' + 'Instead got\n{}'.format(new_x)) + else: + return (x, new_x) + variables = OrderedDict() for name, var in iteritems(obj._variables): if name not in indexers: if var.dtype.kind in 'uifc': - var_indexers = {k: (maybe_variable(obj, k), v) for k, v + var_indexers = {k: _validate_interp_indexer( + maybe_variable(obj, k), v) for k, v in indexers.items() if k in var.dims} variables[name] = missing.interp( var, var_indexers, method, **kwargs) @@ -3810,19 +3829,21 @@ def differentiate(self, coord, edge_order=1, datetime_unit=None): ' dimensional'.format(coord, coord_var.ndim)) dim = coord_var.dims[0] - coord_data = coord_var.data - if coord_data.dtype.kind in 'mM': - if datetime_unit is None: - datetime_unit, _ = np.datetime_data(coord_data.dtype) - coord_data = to_numeric(coord_data, datetime_unit=datetime_unit) + if _contains_datetime_like_objects(coord_var): + if coord_var.dtype.kind in 'mM' and datetime_unit is None: + datetime_unit, _ = np.datetime_data(coord_var.dtype) + elif datetime_unit is None: + datetime_unit = 's' # Default to seconds for cftime objects + coord_var = datetime_to_numeric(coord_var, datetime_unit=datetime_unit) variables = OrderedDict() for k, v in self.variables.items(): if (k in self.data_vars and dim in v.dims and k not in self.coords): - v = to_numeric(v, datetime_unit=datetime_unit) + if _contains_datetime_like_objects(v): + v = datetime_to_numeric(v, datetime_unit=datetime_unit) grad = duck_array_ops.gradient( - v.data, coord_data, edge_order=edge_order, + v.data, coord_var, edge_order=edge_order, axis=v.get_axis_num(dim)) variables[k] = Variable(v.dims, grad) else: diff --git a/xarray/core/missing.py b/xarray/core/missing.py index afb34d99115..0b560c277ae 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -9,9 +9,10 @@ import pandas as pd from . import rolling +from .common import _contains_datetime_like_objects from .computation import apply_ufunc from .pycompat import iteritems -from .utils import is_scalar, OrderedSet, to_numeric +from .utils import is_scalar, OrderedSet, datetime_to_numeric from .variable import Variable, broadcast_variables from .duck_array_ops import dask_array_type @@ -407,15 +408,16 @@ def _floatize_x(x, new_x): x = list(x) new_x = list(new_x) for i in range(len(x)): - if x[i].dtype.kind in 'Mm': + if _contains_datetime_like_objects(x[i]): # Scipy casts coordinates to np.float64, which is not accurate # enough for datetime64 (uses 64bit integer). # We assume that the most of the bits are used to represent the # offset (min(x)) and the variation (x - min(x)) can be # represented by float. - xmin = np.min(x[i]) - x[i] = to_numeric(x[i], offset=xmin, dtype=np.float64) - new_x[i] = to_numeric(new_x[i], offset=xmin, dtype=np.float64) + xmin = x[i].min() + x[i] = datetime_to_numeric(x[i], offset=xmin, dtype=np.float64) + new_x[i] = datetime_to_numeric( + new_x[i], offset=xmin, dtype=np.float64) return x, new_x diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 9d129d5c4f4..c39a07e1b5a 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -593,20 +593,25 @@ def __len__(self): return len(self._data) - num_hidden -def to_numeric(array, offset=None, datetime_unit=None, dtype=float): - """ - Make datetime array float +def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): + """Convert an array containing datetime-like data to an array of floats. + Parameters + ---------- + da : array + Input data offset: Scalar with the same type of array or None If None, subtract minimum values to reduce round off error datetime_unit: None or any of {'Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'ns', 'ps', 'fs', 'as'} dtype: target dtype + + Returns + ------- + array """ - if array.dtype.kind not in ['m', 'M']: - return array.astype(dtype) if offset is None: - offset = np.min(array) + offset = array.min() array = array - offset if datetime_unit: diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index f72c6904f0e..62a29a15247 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -9,10 +9,11 @@ from datetime import timedelta from xarray.coding.cftimeindex import ( parse_iso8601, CFTimeIndex, assert_all_valid_date_type, - _parsed_string_to_bounds, _parse_iso8601_with_reso) + _parsed_string_to_bounds, _parse_iso8601_with_reso, + _parse_array_of_cftime_strings) from xarray.tests import assert_array_equal, assert_identical -from . import has_cftime, has_cftime_or_netCDF4 +from . import has_cftime, has_cftime_or_netCDF4, requires_cftime from .test_coding_times import _all_cftime_date_types @@ -616,3 +617,21 @@ def test_concat_cftimeindex(date_type, enable_cftimeindex): def test_empty_cftimeindex(): index = CFTimeIndex([]) assert index.date_type is None + + +@requires_cftime +def test_parse_array_of_cftime_strings(): + from cftime import DatetimeNoLeap + + strings = np.array(['2000-01-01', '2000-01-02']) + expected = np.array([DatetimeNoLeap(2000, 1, 1), + DatetimeNoLeap(2000, 1, 2)]) + + result = _parse_array_of_cftime_strings(strings, DatetimeNoLeap) + np.testing.assert_array_equal(result, expected) + + # Test scalar array case + strings = np.array('2000-01-01') + expected = np.array(DatetimeNoLeap(2000, 1, 1)) + result = _parse_array_of_cftime_strings(strings, DatetimeNoLeap) + np.testing.assert_array_equal(result, expected) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 237dc09d06a..c42b84c05fc 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -22,8 +22,9 @@ from . import ( InaccessibleArray, TestCase, UnexpectedDataAccess, assert_allclose, - assert_array_equal, assert_equal, assert_identical, has_dask, raises_regex, - requires_bottleneck, requires_dask, requires_scipy, source_ndarray) + assert_array_equal, assert_equal, assert_identical, has_cftime, + has_dask, raises_regex, requires_bottleneck, requires_dask, requires_scipy, + source_ndarray) try: import cPickle as pickle @@ -4524,7 +4525,7 @@ def test_raise_no_warning_for_nan_in_binary_ops(): @pytest.mark.parametrize('dask', [True, False]) @pytest.mark.parametrize('edge_order', [1, 2]) -def test_gradient(dask, edge_order): +def test_differentiate(dask, edge_order): rs = np.random.RandomState(42) coord = [0.2, 0.35, 0.4, 0.6, 0.7, 0.75, 0.76, 0.8] @@ -4562,7 +4563,7 @@ def test_gradient(dask, edge_order): @pytest.mark.parametrize('dask', [True, False]) -def test_gradient_datetime(dask): +def test_differentiate_datetime(dask): rs = np.random.RandomState(42) coord = np.array( ['2004-07-13', '2006-01-13', '2010-08-13', '2010-09-13', @@ -4579,7 +4580,7 @@ def test_gradient_datetime(dask): actual = da.differentiate('x', edge_order=1, datetime_unit='D') expected_x = xr.DataArray( npcompat.gradient( - da, utils.to_numeric(da['x'], datetime_unit='D'), + da, utils.datetime_to_numeric(da['x'], datetime_unit='D'), axis=0, edge_order=1), dims=da.dims, coords=da.coords) assert_equal(expected_x, actual) @@ -4595,3 +4596,32 @@ def test_gradient_datetime(dask): coords={'x': coord}) actual = da.differentiate('x', edge_order=1) assert np.allclose(actual, 1.0) + + +@pytest.mark.skipif(not has_cftime, reason='Test requires cftime.') +@pytest.mark.parametrize('dask', [True, False]) +def test_differentiate_cftime(dask): + rs = np.random.RandomState(42) + coord = xr.cftime_range('2000', periods=8, freq='2M') + + da = xr.DataArray( + rs.randn(8, 6), + coords={'time': coord, 'z': 3, 't2d': (('time', 'y'), rs.randn(8, 6))}, + dims=['time', 'y']) + + if dask and has_dask: + da = da.chunk({'time': 4}) + + actual = da.differentiate('time', edge_order=1, datetime_unit='D') + expected_data = npcompat.gradient( + da, utils.datetime_to_numeric(da['time'], datetime_unit='D'), + axis=0, edge_order=1) + expected = xr.DataArray(expected_data, coords=da.coords, dims=da.dims) + assert_equal(expected, actual) + + actual2 = da.differentiate('time', edge_order=1, datetime_unit='h') + assert_allclose(actual, actual2 * 24) + + # Test the differentiation of datetimes themselves + actual = da['time'].differentiate('time', edge_order=1, datetime_unit='D') + assert_allclose(actual, xr.ones_like(da['time']).astype(float)) diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py index 4a8f4e6eedf..0778a1ff128 100644 --- a/xarray/tests/test_interp.py +++ b/xarray/tests/test_interp.py @@ -5,10 +5,13 @@ import pytest import xarray as xr -from xarray.tests import assert_allclose, assert_equal, requires_scipy +from xarray.tests import (assert_allclose, assert_equal, requires_cftime, + requires_scipy) from . import has_dask, has_scipy from .test_dataset import create_test_data +from ..coding.cftimeindex import _parse_array_of_cftime_strings + try: import scipy except ImportError: @@ -490,3 +493,83 @@ def test_datetime_single_string(): expected = xr.DataArray(0.5) assert_allclose(actual.drop('time'), expected) + + +@requires_cftime +@requires_scipy +def test_cftime(): + times = xr.cftime_range('2000', periods=24, freq='D') + da = xr.DataArray(np.arange(24), coords=[times], dims='time') + + times_new = xr.cftime_range('2000-01-01T12:00:00', periods=3, freq='D') + actual = da.interp(time=times_new) + expected = xr.DataArray([0.5, 1.5, 2.5], coords=[times_new], dims=['time']) + + assert_allclose(actual, expected) + + +@requires_cftime +@requires_scipy +def test_cftime_type_error(): + times = xr.cftime_range('2000', periods=24, freq='D') + da = xr.DataArray(np.arange(24), coords=[times], dims='time') + + times_new = xr.cftime_range('2000-01-01T12:00:00', periods=3, freq='D', + calendar='noleap') + with pytest.raises(TypeError): + da.interp(time=times_new) + + +@requires_cftime +@requires_scipy +def test_cftime_list_of_strings(): + from cftime import DatetimeProlepticGregorian + + times = xr.cftime_range('2000', periods=24, freq='D') + da = xr.DataArray(np.arange(24), coords=[times], dims='time') + + times_new = ['2000-01-01T12:00', '2000-01-02T12:00', '2000-01-03T12:00'] + actual = da.interp(time=times_new) + + times_new_array = _parse_array_of_cftime_strings( + np.array(times_new), DatetimeProlepticGregorian) + expected = xr.DataArray([0.5, 1.5, 2.5], coords=[times_new_array], + dims=['time']) + + assert_allclose(actual, expected) + + +@requires_cftime +@requires_scipy +def test_cftime_single_string(): + from cftime import DatetimeProlepticGregorian + + times = xr.cftime_range('2000', periods=24, freq='D') + da = xr.DataArray(np.arange(24), coords=[times], dims='time') + + times_new = '2000-01-01T12:00' + actual = da.interp(time=times_new) + + times_new_array = _parse_array_of_cftime_strings( + np.array(times_new), DatetimeProlepticGregorian) + expected = xr.DataArray(0.5, coords={'time': times_new_array}) + + assert_allclose(actual, expected) + + +@requires_scipy +def test_datetime_to_non_datetime_error(): + da = xr.DataArray(np.arange(24), dims='time', + coords={'time': pd.date_range('2000-01-01', periods=24)}) + with pytest.raises(TypeError): + da.interp(time=0.5) + + +@requires_cftime +@requires_scipy +def test_cftime_to_non_cftime_error(): + times = xr.cftime_range('2000', periods=24, freq='D') + da = xr.DataArray(np.arange(24), coords=[times], dims='time') + + with pytest.raises(TypeError): + da.interp(time=0.5) diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index ed8045b78e4..0c0e0f3f744 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -5,16 +5,18 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from xarray.coding.cftimeindex import CFTimeIndex from xarray.core import duck_array_ops, utils from xarray.core.options import set_options from xarray.core.pycompat import OrderedDict from xarray.core.utils import either_dict_or_kwargs +from xarray.testing import assert_identical from . import ( TestCase, assert_array_equal, has_cftime, has_cftime_or_netCDF4, - requires_dask) + requires_dask, requires_cftime) from .test_coding_times import _all_cftime_date_types @@ -263,3 +265,42 @@ def test_either_dict_or_kwargs(): with pytest.raises(ValueError, match=r'foo'): result = either_dict_or_kwargs(dict(a=1), dict(a=1), 'foo') + + +def test_datetime_to_numeric_datetime64(): + times = pd.date_range('2000', periods=5, freq='7D') + da = xr.DataArray(times, coords=[times], dims=['time']) + result = utils.datetime_to_numeric(da, datetime_unit='h') + expected = 24 * xr.DataArray(np.arange(0, 35, 7), coords=da.coords) + assert_identical(result, expected) + + offset = da.isel(time=1) + result = utils.datetime_to_numeric(da, offset=offset, datetime_unit='h') + expected = 24 * xr.DataArray(np.arange(-7, 28, 7), coords=da.coords) + assert_identical(result, expected) + + dtype = np.float32 + result = utils.datetime_to_numeric(da, datetime_unit='h', dtype=dtype) + expected = 24 * xr.DataArray( + np.arange(0, 35, 7), coords=da.coords).astype(dtype) + assert_identical(result, expected) + + +@requires_cftime +def test_datetime_to_numeric_cftime(): + times = xr.cftime_range('2000', periods=5, freq='7D') + da = xr.DataArray(times, coords=[times], dims=['time']) + result = utils.datetime_to_numeric(da, datetime_unit='h') + expected = 24 * xr.DataArray(np.arange(0, 35, 7), coords=da.coords) + assert_identical(result, expected) + + offset = da.isel(time=1) + result = utils.datetime_to_numeric(da, offset=offset, datetime_unit='h') + expected = 24 * xr.DataArray(np.arange(-7, 28, 7), coords=da.coords) + assert_identical(result, expected) + + dtype = np.float32 + result = utils.datetime_to_numeric(da, datetime_unit='h', dtype=dtype) + expected = 24 * xr.DataArray( + np.arange(0, 35, 7), coords=da.coords).astype(dtype) + assert_identical(result, expected) From 23d1cda3b7da5c73a5f561a5c953b50beaa2bfe6 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 28 Sep 2018 20:24:35 +0200 Subject: [PATCH 230/282] fix:2445 (#2446) * fix:2445 * rename test_shift_multidim -> test_roll_multidim --- doc/whats-new.rst | 4 ++++ xarray/core/dataset.py | 3 ++- xarray/tests/test_dataset.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index a5b7b36142e..8b145924f2d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -56,6 +56,10 @@ Enhancements Bug fixes ~~~~~~~~~ +- ``xarray.DataArray.roll`` correctly handles multidimensional arrays. + (:issue:`2445`) + By `Keisuke Fujii `_. + - ``xarray.DataArray.std()`` now correctly accepts ``ddof`` keyword argument. (:issue:`2240`) By `Keisuke Fujii `_. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 4ad5902ebad..5e787c1587b 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -3595,7 +3595,8 @@ def roll(self, shifts=None, roll_coords=None, **shifts_kwargs): variables = OrderedDict() for k, v in iteritems(self.variables): if k not in unrolled_vars: - variables[k] = v.roll(**shifts) + variables[k] = v.roll(**{k: s for k, s in shifts.items() + if k in v.dims}) else: variables[k] = v diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index c42b84c05fc..2c964b81b98 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -3945,6 +3945,16 @@ def test_roll_coords_none(self): expected = Dataset({'foo': ('x', [3, 1, 2])}, ex_coords, attrs) assert_identical(expected, actual) + def test_roll_multidim(self): + # regression test for 2445 + arr = xr.DataArray( + [[1, 2, 3],[4, 5, 6]], coords={'x': range(3), 'y': range(2)}, + dims=('y','x')) + actual = arr.roll(x=1, roll_coords=True) + expected = xr.DataArray([[3, 1, 2],[6, 4, 5]], + coords=[('y', [0, 1]), ('x', [2, 0, 1])]) + assert_identical(expected, actual) + def test_real_and_imag(self): attrs = {'foo': 'bar'} ds = Dataset({'x': ((), 1 + 2j, attrs)}, attrs=attrs) From f9c4169150286fa1aac020ab965380ed21fe1148 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 30 Sep 2018 09:16:48 -0400 Subject: [PATCH 231/282] Fix FutureWarning in CFTimeIndex.date_type (#2448) --- xarray/coding/cftimeindex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index e236dca3693..faf1a044505 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -157,7 +157,7 @@ def f(self): def get_date_type(self): - if self.data: + if self._data.size: return type(self._data[0]) else: return None From 8fb57f7b9ff683225650a928b8d7d287d8954e79 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Tue, 2 Oct 2018 10:44:29 -0400 Subject: [PATCH 232/282] Add CFTimeIndex.shift (#2431) * Add CFTimeIndex.shift Update what's new Add bug fix note * Add example to docstring * Use pycompat for basestring * Generate an API reference page for CFTimeIndex.shift --- doc/api-hidden.rst | 2 + doc/whats-new.rst | 5 +++ xarray/coding/cftimeindex.py | 51 ++++++++++++++++++++++++ xarray/tests/test_cftimeindex.py | 66 ++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) diff --git a/doc/api-hidden.rst b/doc/api-hidden.rst index 1826cc86892..0e8143c72ea 100644 --- a/doc/api-hidden.rst +++ b/doc/api-hidden.rst @@ -151,3 +151,5 @@ plot.FacetGrid.set_titles plot.FacetGrid.set_ticks plot.FacetGrid.map + + CFTimeIndex.shift diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8b145924f2d..3f8d40910cd 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -46,6 +46,9 @@ Enhancements - Added support for Python 3.7. (:issue:`2271`). By `Joe Hamman `_. +- Added :py:meth:`~xarray.CFTimeIndex.shift` for shifting the values of a + CFTimeIndex by a specified frequency. (:issue:`2244`). By `Spencer Clark + `_. - Added support for using ``cftime.datetime`` coordinates with :py:meth:`~xarray.DataArray.differentiate`, :py:meth:`~xarray.Dataset.differentiate`, @@ -56,6 +59,8 @@ Enhancements Bug fixes ~~~~~~~~~ +- Addition and subtraction operators used with a CFTimeIndex now preserve the + index's type. (:issue:`2244`). By `Spencer Clark `_. - ``xarray.DataArray.roll`` correctly handles multidimensional arrays. (:issue:`2445`) By `Keisuke Fujii `_. diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index faf1a044505..341ecfed262 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -315,6 +315,57 @@ def contains(self, key): """Needed for .loc based partial-string indexing""" return self.__contains__(key) + def shift(self, n, freq): + """Shift the CFTimeIndex a multiple of the given frequency. + + See the documentation for :py:func:`~xarray.cftime_range` for a + complete listing of valid frequency strings. + + Parameters + ---------- + n : int + Periods to shift by + freq : str or datetime.timedelta + A frequency string or datetime.timedelta object to shift by + + Returns + ------- + CFTimeIndex + + See also + -------- + pandas.DatetimeIndex.shift + + Examples + -------- + >>> index = xr.cftime_range('2000', periods=1, freq='M') + >>> index + CFTimeIndex([2000-01-31 00:00:00], dtype='object') + >>> index.shift(1, 'M') + CFTimeIndex([2000-02-29 00:00:00], dtype='object') + """ + from .cftime_offsets import to_offset + + if not isinstance(n, int): + raise TypeError("'n' must be an int, got {}.".format(n)) + if isinstance(freq, timedelta): + return self + n * freq + elif isinstance(freq, pycompat.basestring): + return self + n * to_offset(freq) + else: + raise TypeError( + "'freq' must be of type " + "str or datetime.timedelta, got {}.".format(freq)) + + def __add__(self, other): + return CFTimeIndex(np.array(self) + other) + + def __radd__(self, other): + return CFTimeIndex(other + np.array(self)) + + def __sub__(self, other): + return CFTimeIndex(np.array(self) - other) + def _parse_iso8601_without_reso(date_type, datetime_str): date, _ = _parse_iso8601_with_reso(date_type, datetime_str) diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index 62a29a15247..a558ab9a784 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -619,6 +619,72 @@ def test_empty_cftimeindex(): assert index.date_type is None +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_cftimeindex_add(index): + date_type = index.date_type + expected_dates = [date_type(1, 1, 2), date_type(1, 2, 2), + date_type(2, 1, 2), date_type(2, 2, 2)] + expected = CFTimeIndex(expected_dates) + result = index + timedelta(days=1) + assert result.equals(expected) + assert isinstance(result, CFTimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_cftimeindex_radd(index): + date_type = index.date_type + expected_dates = [date_type(1, 1, 2), date_type(1, 2, 2), + date_type(2, 1, 2), date_type(2, 2, 2)] + expected = CFTimeIndex(expected_dates) + result = timedelta(days=1) + index + assert result.equals(expected) + assert isinstance(result, CFTimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_cftimeindex_sub(index): + date_type = index.date_type + expected_dates = [date_type(1, 1, 2), date_type(1, 2, 2), + date_type(2, 1, 2), date_type(2, 2, 2)] + expected = CFTimeIndex(expected_dates) + result = index + timedelta(days=2) + result = result - timedelta(days=1) + assert result.equals(expected) + assert isinstance(result, CFTimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_cftimeindex_rsub(index): + with pytest.raises(TypeError): + timedelta(days=1) - index + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('freq', ['D', timedelta(days=1)]) +def test_cftimeindex_shift(index, freq): + date_type = index.date_type + expected_dates = [date_type(1, 1, 3), date_type(1, 2, 3), + date_type(2, 1, 3), date_type(2, 2, 3)] + expected = CFTimeIndex(expected_dates) + result = index.shift(2, freq) + assert result.equals(expected) + assert isinstance(result, CFTimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_cftimeindex_shift_invalid_n(): + index = xr.cftime_range('2000', periods=3) + with pytest.raises(TypeError): + index.shift('a', 'D') + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_cftimeindex_shift_invalid_freq(): + index = xr.cftime_range('2000', periods=3) + with pytest.raises(TypeError): + index.shift(1, 1) + + @requires_cftime def test_parse_array_of_cftime_strings(): from cftime import DatetimeNoLeap From 1e7a1d348d927bfa4fd4fba58a3f7600314746cf Mon Sep 17 00:00:00 2001 From: Denis Rykov Date: Tue, 2 Oct 2018 17:05:25 +0200 Subject: [PATCH 233/282] np.AxisError was added in numpy 1.13 (#2455) --- xarray/core/dask_array_compat.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xarray/core/dask_array_compat.py b/xarray/core/dask_array_compat.py index 5e6b81a253d..2196dba7f86 100644 --- a/xarray/core/dask_array_compat.py +++ b/xarray/core/dask_array_compat.py @@ -44,7 +44,13 @@ def isin(element, test_elements, assume_unique=False, invert=False): import math from numbers import Integral, Real - AxisError = np.AxisError + try: + AxisError = np.AxisError + except AttributeError: + try: + np.array([0]).sum(axis=5) + except Exception as e: + AxisError = type(e) def validate_axis(axis, ndim): """ Validate an input to axis= keywords """ From 0f70a876759197388d32d6d9f0317f0fe63e0336 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 3 Oct 2018 00:45:59 +0900 Subject: [PATCH 234/282] plot.contour: Don't make cmap if colors is a single color. (#2453) By default, matplotlib draw dashed negative contours for a single color. We lost this feature by manually specifying cmap everytime. --- doc/whats-new.rst | 4 ++++ xarray/plot/plot.py | 5 +++++ xarray/tests/test_plot.py | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3f8d40910cd..8e0526f8b8b 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -68,6 +68,10 @@ Bug fixes - ``xarray.DataArray.std()`` now correctly accepts ``ddof`` keyword argument. (:issue:`2240`) By `Keisuke Fujii `_. +- Restore matplotlib's default of plotting dashed negative contours when + a single color is passed to ``DataArray.contour()`` e.g. ``colors='k'``. + By `Deepak Cherian `_. + .. _whats-new.0.10.9: diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index a6add44682f..3f9f1090c70 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -737,6 +737,11 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, # pcolormesh kwargs['extend'] = cmap_params['extend'] kwargs['levels'] = cmap_params['levels'] + # if colors == a single color, matplotlib draws dashed negative + # contours. we lose this feature if we pass cmap and not colors + if isinstance(colors, basestring): + cmap_params['cmap'] = None + kwargs['colors'] = colors if 'pcolormesh' == plotfunc.__name__: kwargs['infer_intervals'] = infer_intervals diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 1423f7ae853..98265149122 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1140,9 +1140,9 @@ def test_colors(self): def _color_as_tuple(c): return tuple(c[:3]) + # with single color, we don't want rgb array artist = self.plotmethod(colors='k') - assert _color_as_tuple(artist.cmap.colors[0]) == \ - (0.0, 0.0, 0.0) + assert artist.cmap.colors[0] == 'k' artist = self.plotmethod(colors=['k', 'b']) assert _color_as_tuple(artist.cmap.colors[1]) == \ From 3cef8d730d5bbd699a393fa15266064ebb9849e2 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 5 Oct 2018 04:02:17 -0400 Subject: [PATCH 235/282] Clean up _parse_array_of_cftime_strings (#2464) * Make _parse_array_of_cftime_strings more robust * lint --- xarray/coding/cftimeindex.py | 7 ++----- xarray/tests/test_cftimeindex.py | 8 +++++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 341ecfed262..75a1fc9bd1a 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -389,8 +389,5 @@ def _parse_array_of_cftime_strings(strings, date_type): ------- np.array """ - if strings.ndim == 0: - return np.array(_parse_iso8601_without_reso(date_type, strings.item())) - else: - return np.array([_parse_iso8601_without_reso(date_type, s) - for s in strings]) + return np.array([_parse_iso8601_without_reso(date_type, s) + for s in strings.ravel()]).reshape(strings.shape) diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index a558ab9a784..33bf2cbce0d 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -689,9 +689,11 @@ def test_cftimeindex_shift_invalid_freq(): def test_parse_array_of_cftime_strings(): from cftime import DatetimeNoLeap - strings = np.array(['2000-01-01', '2000-01-02']) - expected = np.array([DatetimeNoLeap(2000, 1, 1), - DatetimeNoLeap(2000, 1, 2)]) + strings = np.array([['2000-01-01', '2000-01-02'], + ['2000-01-03', '2000-01-04']]) + expected = np.array( + [[DatetimeNoLeap(2000, 1, 1), DatetimeNoLeap(2000, 1, 2)], + [DatetimeNoLeap(2000, 1, 3), DatetimeNoLeap(2000, 1, 4)]]) result = _parse_array_of_cftime_strings(strings, DatetimeNoLeap) np.testing.assert_array_equal(result, expected) From 3f697fe013dc510cebb6a64d0a2c760d6320573a Mon Sep 17 00:00:00 2001 From: Jie Chen Date: Fri, 5 Oct 2018 08:35:56 -0700 Subject: [PATCH 236/282] Update whats-new.rst (#2466) --- doc/whats-new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4a6886159d1..662d60e29e7 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,7 +27,7 @@ What's New .. _whats-new.0.10.9: -v0.10.9 (21 September 2019) +v0.10.9 (21 September 2018) --------------------------- This minor release contains a number of backwards compatible enhancements. From 4a7a103204989af7e2b6bc97a4109d81beebd34c Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sat, 6 Oct 2018 02:48:57 -0500 Subject: [PATCH 237/282] Add python_requires to setup.py (#2465) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 88c27c95118..68798bdf219 100644 --- a/setup.py +++ b/setup.py @@ -69,5 +69,6 @@ install_requires=INSTALL_REQUIRES, tests_require=TESTS_REQUIRE, url=URL, + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*', packages=find_packages(), package_data={'xarray': ['tests/data/*']}) From bb87a9441d22b390e069d0fde58f297a054fd98a Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 6 Oct 2018 13:09:13 -0400 Subject: [PATCH 238/282] Replace the last of unittest with pytest (#2467) * cleaning * remove assertEqual * remove assertItems * more removing assertitems * remove assertequal * remove TestCase * straggler * pep8replies * pep8replies2 * small flups * pytest.warns requires Warning class * tuple list comparisons * disable test check * set / list * the last unittest survivor, and automated formatting changes where possible --- xarray/tests/__init__.py | 40 +--- xarray/tests/test_accessors.py | 9 +- xarray/tests/test_backends.py | 281 ++++++++++++++-------------- xarray/tests/test_combine.py | 10 +- xarray/tests/test_conventions.py | 23 +-- xarray/tests/test_dask.py | 12 +- xarray/tests/test_dataarray.py | 64 ++++--- xarray/tests/test_dataset.py | 81 ++++---- xarray/tests/test_duck_array_ops.py | 12 +- xarray/tests/test_extensions.py | 4 +- xarray/tests/test_formatting.py | 8 +- xarray/tests/test_indexing.py | 18 +- xarray/tests/test_merge.py | 10 +- xarray/tests/test_plot.py | 61 +++--- xarray/tests/test_tutorial.py | 8 +- xarray/tests/test_utils.py | 23 +-- xarray/tests/test_variable.py | 48 ++--- 17 files changed, 346 insertions(+), 366 deletions(-) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 33a8da6bbfb..285c1f03a26 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -9,11 +9,10 @@ import numpy as np from numpy.testing import assert_array_equal # noqa: F401 -from xarray.core.duck_array_ops import allclose_or_equiv +from xarray.core.duck_array_ops import allclose_or_equiv # noqa import pytest from xarray.core import utils -from xarray.core.pycompat import PY3 from xarray.core.indexing import ExplicitlyIndexed from xarray.testing import (assert_equal, assert_identical, # noqa: F401 assert_allclose) @@ -25,10 +24,6 @@ # old location, for pandas < 0.20 from pandas.util.testing import assert_frame_equal # noqa: F401 -try: - import unittest2 as unittest -except ImportError: - import unittest try: from unittest import mock @@ -117,39 +112,6 @@ def _importorskip(modname, minversion=None): "internet connection") -class TestCase(unittest.TestCase): - """ - These functions are all deprecated. Instead, use functions in xr.testing - """ - if PY3: - # Python 3 assertCountEqual is roughly equivalent to Python 2 - # assertItemsEqual - def assertItemsEqual(self, first, second, msg=None): - __tracebackhide__ = True # noqa: F841 - return self.assertCountEqual(first, second, msg) - - @contextmanager - def assertWarns(self, message): - __tracebackhide__ = True # noqa: F841 - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', message) - yield - assert len(w) > 0 - assert any(message in str(wi.message) for wi in w) - - def assertVariableNotEqual(self, v1, v2): - __tracebackhide__ = True # noqa: F841 - assert not v1.equals(v2) - - def assertEqual(self, a1, a2): - __tracebackhide__ = True # noqa: F841 - assert a1 == a2 or (a1 != a1 and a2 != a2) - - def assertAllClose(self, a1, a2, rtol=1e-05, atol=1e-8): - __tracebackhide__ = True # noqa: F841 - assert allclose_or_equiv(a1, a2, rtol=rtol, atol=atol) - - @contextmanager def raises_regex(error, pattern): __tracebackhide__ = True # noqa: F841 diff --git a/xarray/tests/test_accessors.py b/xarray/tests/test_accessors.py index e1b3a95b942..38038fc8f65 100644 --- a/xarray/tests/test_accessors.py +++ b/xarray/tests/test_accessors.py @@ -7,12 +7,13 @@ import xarray as xr from . import ( - TestCase, assert_array_equal, assert_equal, raises_regex, requires_dask, - has_cftime, has_dask, has_cftime_or_netCDF4) + assert_array_equal, assert_equal, has_cftime, has_cftime_or_netCDF4, + has_dask, raises_regex, requires_dask) -class TestDatetimeAccessor(TestCase): - def setUp(self): +class TestDatetimeAccessor(object): + @pytest.fixture(autouse=True) + def setup(self): nt = 100 data = np.random.rand(10, 10, nt) lons = np.linspace(0, 11, 10) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 8b469761ccd..a2e1cb4c0fa 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2,6 +2,7 @@ import contextlib import itertools +import math import os.path import pickle import shutil @@ -19,8 +20,8 @@ from xarray import ( DataArray, Dataset, backends, open_dataarray, open_dataset, open_mfdataset, save_mfdataset) -from xarray.backends.common import (robust_getitem, - PickleByReconstructionWrapper) +from xarray.backends.common import ( + PickleByReconstructionWrapper, robust_getitem) from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.backends.pydap_ import PydapDataStore from xarray.core import indexing @@ -29,12 +30,11 @@ from xarray.tests import mock from . import ( - TestCase, assert_allclose, assert_array_equal, assert_equal, - assert_identical, has_dask, has_netCDF4, has_scipy, network, raises_regex, + assert_allclose, assert_array_equal, assert_equal, assert_identical, + has_dask, has_netCDF4, has_scipy, network, raises_regex, requires_cftime, requires_dask, requires_h5netcdf, requires_netCDF4, requires_pathlib, - requires_pydap, requires_pynio, requires_rasterio, requires_scipy, - requires_scipy_or_netCDF4, requires_zarr, requires_pseudonetcdf, - requires_cftime) + requires_pseudonetcdf, requires_pydap, requires_pynio, requires_rasterio, + requires_scipy, requires_scipy_or_netCDF4, requires_zarr) from .test_dataset import create_test_data try: @@ -106,7 +106,7 @@ def create_boolean_data(): return Dataset({'x': ('t', [True, False, False, True], attributes)}) -class TestCommon(TestCase): +class TestCommon(object): def test_robust_getitem(self): class UnreliableArrayFailure(Exception): @@ -126,11 +126,11 @@ def __getitem__(self, key): array = UnreliableArray([0]) with pytest.raises(UnreliableArrayFailure): array[0] - self.assertEqual(array[0], 0) + assert array[0] == 0 actual = robust_getitem(array, 0, catch=UnreliableArrayFailure, initial_delay=0) - self.assertEqual(actual, 0) + assert actual == 0 class NetCDF3Only(object): @@ -222,11 +222,11 @@ def assert_loads(vars=None): with self.roundtrip(expected) as actual: for k, v in actual.variables.items(): # IndexVariables are eagerly loaded into memory - self.assertEqual(v._in_memory, k in actual.dims) + assert v._in_memory == (k in actual.dims) yield actual for k, v in actual.variables.items(): if k in vars: - self.assertTrue(v._in_memory) + assert v._in_memory assert_identical(expected, actual) with pytest.raises(AssertionError): @@ -252,14 +252,14 @@ def test_dataset_compute(self): # Test Dataset.compute() for k, v in actual.variables.items(): # IndexVariables are eagerly cached - self.assertEqual(v._in_memory, k in actual.dims) + assert v._in_memory == (k in actual.dims) computed = actual.compute() for k, v in actual.variables.items(): - self.assertEqual(v._in_memory, k in actual.dims) + assert v._in_memory == (k in actual.dims) for v in computed.variables.values(): - self.assertTrue(v._in_memory) + assert v._in_memory assert_identical(expected, actual) assert_identical(expected, computed) @@ -343,12 +343,12 @@ def test_roundtrip_string_encoded_characters(self): expected['x'].encoding['dtype'] = 'S1' with self.roundtrip(expected) as actual: assert_identical(expected, actual) - self.assertEqual(actual['x'].encoding['_Encoding'], 'utf-8') + assert actual['x'].encoding['_Encoding'] == 'utf-8' expected['x'].encoding['_Encoding'] = 'ascii' with self.roundtrip(expected) as actual: assert_identical(expected, actual) - self.assertEqual(actual['x'].encoding['_Encoding'], 'ascii') + assert actual['x'].encoding['_Encoding'] == 'ascii' def test_roundtrip_numpy_datetime_data(self): times = pd.to_datetime(['2000-01-01', '2000-01-02', 'NaT']) @@ -434,10 +434,10 @@ def test_roundtrip_coordinates_with_space(self): def test_roundtrip_boolean_dtype(self): original = create_boolean_data() - self.assertEqual(original['x'].dtype, 'bool') + assert original['x'].dtype == 'bool' with self.roundtrip(original) as actual: assert_identical(original, actual) - self.assertEqual(actual['x'].dtype, 'bool') + assert actual['x'].dtype == 'bool' def test_orthogonal_indexing(self): in_memory = create_test_data() @@ -626,20 +626,20 @@ def test_unsigned_roundtrip_mask_and_scale(self): encoded = create_encoded_unsigned_masked_scaled_data() with self.roundtrip(decoded) as actual: for k in decoded.variables: - self.assertEqual(decoded.variables[k].dtype, - actual.variables[k].dtype) + assert (decoded.variables[k].dtype == + actual.variables[k].dtype) assert_allclose(decoded, actual, decode_bytes=False) with self.roundtrip(decoded, open_kwargs=dict(decode_cf=False)) as actual: for k in encoded.variables: - self.assertEqual(encoded.variables[k].dtype, - actual.variables[k].dtype) + assert (encoded.variables[k].dtype == + actual.variables[k].dtype) assert_allclose(encoded, actual, decode_bytes=False) with self.roundtrip(encoded, open_kwargs=dict(decode_cf=False)) as actual: for k in encoded.variables: - self.assertEqual(encoded.variables[k].dtype, - actual.variables[k].dtype) + assert (encoded.variables[k].dtype == + actual.variables[k].dtype) assert_allclose(encoded, actual, decode_bytes=False) # make sure roundtrip encoding didn't change the # original dataset. @@ -647,14 +647,14 @@ def test_unsigned_roundtrip_mask_and_scale(self): encoded, create_encoded_unsigned_masked_scaled_data()) with self.roundtrip(encoded) as actual: for k in decoded.variables: - self.assertEqual(decoded.variables[k].dtype, - actual.variables[k].dtype) + assert decoded.variables[k].dtype == \ + actual.variables[k].dtype assert_allclose(decoded, actual, decode_bytes=False) with self.roundtrip(encoded, open_kwargs=dict(decode_cf=False)) as actual: for k in encoded.variables: - self.assertEqual(encoded.variables[k].dtype, - actual.variables[k].dtype) + assert encoded.variables[k].dtype == \ + actual.variables[k].dtype assert_allclose(encoded, actual, decode_bytes=False) def test_roundtrip_mask_and_scale(self): @@ -692,12 +692,11 @@ def equals_latlon(obj): with create_tmp_file() as tmp_file: original.to_netcdf(tmp_file) with open_dataset(tmp_file, decode_coords=False) as ds: - self.assertTrue(equals_latlon(ds['temp'].attrs['coordinates'])) - self.assertTrue( - equals_latlon(ds['precip'].attrs['coordinates'])) - self.assertNotIn('coordinates', ds.attrs) - self.assertNotIn('coordinates', ds['lat'].attrs) - self.assertNotIn('coordinates', ds['lon'].attrs) + assert equals_latlon(ds['temp'].attrs['coordinates']) + assert equals_latlon(ds['precip'].attrs['coordinates']) + assert 'coordinates' not in ds.attrs + assert 'coordinates' not in ds['lat'].attrs + assert 'coordinates' not in ds['lon'].attrs modified = original.drop(['temp', 'precip']) with self.roundtrip(modified) as actual: @@ -705,9 +704,9 @@ def equals_latlon(obj): with create_tmp_file() as tmp_file: modified.to_netcdf(tmp_file) with open_dataset(tmp_file, decode_coords=False) as ds: - self.assertTrue(equals_latlon(ds.attrs['coordinates'])) - self.assertNotIn('coordinates', ds['lat'].attrs) - self.assertNotIn('coordinates', ds['lon'].attrs) + assert equals_latlon(ds.attrs['coordinates']) + assert 'coordinates' not in ds['lat'].attrs + assert 'coordinates' not in ds['lon'].attrs def test_roundtrip_endian(self): ds = Dataset({'x': np.arange(3, 10, dtype='>i2'), @@ -743,8 +742,8 @@ def test_encoding_kwarg(self): ds = Dataset({'x': ('y', np.arange(10.0))}) kwargs = dict(encoding={'x': {'dtype': 'f4'}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertEqual(actual.x.encoding['dtype'], 'f4') - self.assertEqual(ds.x.encoding, {}) + assert actual.x.encoding['dtype'] == 'f4' + assert ds.x.encoding == {} kwargs = dict(encoding={'x': {'foo': 'bar'}}) with raises_regex(ValueError, 'unexpected encoding'): @@ -766,7 +765,7 @@ def test_encoding_kwarg_dates(self): units = 'days since 1900-01-01' kwargs = dict(encoding={'t': {'units': units}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertEqual(actual.t.encoding['units'], units) + assert actual.t.encoding['units'] == units assert_identical(actual, ds) def test_encoding_kwarg_fixed_width_string(self): @@ -778,7 +777,7 @@ def test_encoding_kwarg_fixed_width_string(self): ds = Dataset({'x': strings}) kwargs = dict(encoding={'x': {'dtype': 'S1'}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertEqual(actual['x'].encoding['dtype'], 'S1') + assert actual['x'].encoding['dtype'] == 'S1' assert_identical(actual, ds) def test_default_fill_value(self): @@ -786,9 +785,8 @@ def test_default_fill_value(self): ds = Dataset({'x': ('y', np.arange(10.0))}) kwargs = dict(encoding={'x': {'dtype': 'f4'}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertEqual(actual.x.encoding['_FillValue'], - np.nan) - self.assertEqual(ds.x.encoding, {}) + assert math.isnan(actual.x.encoding['_FillValue']) + assert ds.x.encoding == {} # Test default encoding for int: ds = Dataset({'x': ('y', np.arange(10.0))}) @@ -797,14 +795,14 @@ def test_default_fill_value(self): warnings.filterwarnings( 'ignore', '.*floating point data as an integer') with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertTrue('_FillValue' not in actual.x.encoding) - self.assertEqual(ds.x.encoding, {}) + assert '_FillValue' not in actual.x.encoding + assert ds.x.encoding == {} # Test default encoding for implicit int: ds = Dataset({'x': ('y', np.arange(10, dtype='int16'))}) with self.roundtrip(ds) as actual: - self.assertTrue('_FillValue' not in actual.x.encoding) - self.assertEqual(ds.x.encoding, {}) + assert '_FillValue' not in actual.x.encoding + assert ds.x.encoding == {} def test_explicitly_omit_fill_value(self): ds = Dataset({'x': ('y', [np.pi, -np.pi])}) @@ -817,7 +815,7 @@ def test_explicitly_omit_fill_value_via_encoding_kwarg(self): kwargs = dict(encoding={'x': {'_FillValue': None}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: assert '_FillValue' not in actual.x.encoding - self.assertEqual(ds.y.encoding, {}) + assert ds.y.encoding == {} def test_explicitly_omit_fill_value_in_coord(self): ds = Dataset({'x': ('y', [np.pi, -np.pi])}, coords={'y': [0.0, 1.0]}) @@ -830,14 +828,14 @@ def test_explicitly_omit_fill_value_in_coord_via_encoding_kwarg(self): kwargs = dict(encoding={'y': {'_FillValue': None}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: assert '_FillValue' not in actual.y.encoding - self.assertEqual(ds.y.encoding, {}) + assert ds.y.encoding == {} def test_encoding_same_dtype(self): ds = Dataset({'x': ('y', np.arange(10.0, dtype='f4'))}) kwargs = dict(encoding={'x': {'dtype': 'f4'}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertEqual(actual.x.encoding['dtype'], 'f4') - self.assertEqual(ds.x.encoding, {}) + assert actual.x.encoding['dtype'] == 'f4' + assert ds.x.encoding == {} def test_append_write(self): # regression for GH1215 @@ -1015,7 +1013,7 @@ def test_default_to_char_arrays(self): data = Dataset({'x': np.array(['foo', 'zzzz'], dtype='S')}) with self.roundtrip(data) as actual: assert_identical(data, actual) - self.assertEqual(actual['x'].dtype, np.dtype('S4')) + assert actual['x'].dtype == np.dtype('S4') def test_open_encodings(self): # Create a netCDF file with explicit time units @@ -1040,15 +1038,15 @@ def test_open_encodings(self): actual_encoding = dict((k, v) for k, v in iteritems(actual['time'].encoding) if k in expected['time'].encoding) - self.assertDictEqual(actual_encoding, - expected['time'].encoding) + assert actual_encoding == \ + expected['time'].encoding def test_dump_encodings(self): # regression test for #709 ds = Dataset({'x': ('y', np.arange(10.0))}) kwargs = dict(encoding={'x': {'zlib': True}}) with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertTrue(actual.x.encoding['zlib']) + assert actual.x.encoding['zlib'] def test_dump_and_open_encodings(self): # Create a netCDF file with explicit time units @@ -1066,8 +1064,7 @@ def test_dump_and_open_encodings(self): with create_tmp_file() as tmp_file2: xarray_dataset.to_netcdf(tmp_file2) with nc4.Dataset(tmp_file2, 'r') as ds: - self.assertEqual( - ds.variables['time'].getncattr('units'), units) + assert ds.variables['time'].getncattr('units') == units assert_array_equal( ds.variables['time'], np.arange(10) + 4) @@ -1080,7 +1077,7 @@ def test_compression_encoding(self): 'original_shape': data.var2.shape}) with self.roundtrip(data) as actual: for k, v in iteritems(data['var2'].encoding): - self.assertEqual(v, actual['var2'].encoding[k]) + assert v == actual['var2'].encoding[k] # regression test for #156 expected = data.isel(dim1=0) @@ -1095,14 +1092,14 @@ def test_encoding_kwarg_compression(self): with self.roundtrip(ds, save_kwargs=kwargs) as actual: assert_equal(actual, ds) - self.assertEqual(actual.x.encoding['dtype'], 'f4') - self.assertEqual(actual.x.encoding['zlib'], True) - self.assertEqual(actual.x.encoding['complevel'], 9) - self.assertEqual(actual.x.encoding['fletcher32'], True) - self.assertEqual(actual.x.encoding['chunksizes'], (5,)) - self.assertEqual(actual.x.encoding['shuffle'], True) + assert actual.x.encoding['dtype'] == 'f4' + assert actual.x.encoding['zlib'] + assert actual.x.encoding['complevel'] == 9 + assert actual.x.encoding['fletcher32'] + assert actual.x.encoding['chunksizes'] == (5,) + assert actual.x.encoding['shuffle'] - self.assertEqual(ds.x.encoding, {}) + assert ds.x.encoding == {} def test_encoding_chunksizes_unlimited(self): # regression test for GH1225 @@ -1183,7 +1180,7 @@ def test_read_variable_len_strings(self): @requires_netCDF4 -class NetCDF4DataTest(BaseNetCDF4Test, TestCase): +class NetCDF4DataTest(BaseNetCDF4Test): autoclose = False @contextlib.contextmanager @@ -1201,7 +1198,7 @@ def test_variable_order(self): ds.coords['c'] = 4 with self.roundtrip(ds) as actual: - self.assertEqual(list(ds.variables), list(actual.variables)) + assert list(ds.variables) == list(actual.variables) def test_unsorted_index_raises(self): # should be fixed in netcdf4 v1.2.1 @@ -1220,7 +1217,7 @@ def test_unsorted_index_raises(self): try: ds2.randovar.values except IndexError as err: - self.assertIn('first by calling .load', str(err)) + assert 'first by calling .load' in str(err) def test_88_character_filename_segmentation_fault(self): # should be fixed in netcdf4 v1.3.1 @@ -1335,17 +1332,17 @@ def test_auto_chunk(self): original, open_kwargs={'auto_chunk': False}) as actual: for k, v in actual.variables.items(): # only index variables should be in memory - self.assertEqual(v._in_memory, k in actual.dims) + assert v._in_memory == (k in actual.dims) # there should be no chunks - self.assertEqual(v.chunks, None) + assert v.chunks is None with self.roundtrip( original, open_kwargs={'auto_chunk': True}) as actual: for k, v in actual.variables.items(): # only index variables should be in memory - self.assertEqual(v._in_memory, k in actual.dims) + assert v._in_memory == (k in actual.dims) # chunk size should be the same as original - self.assertEqual(v.chunks, original[k].chunks) + assert v.chunks == original[k].chunks def test_write_uneven_dask_chunks(self): # regression for GH#2225 @@ -1365,7 +1362,7 @@ def test_chunk_encoding(self): data['var2'].encoding.update({'chunks': chunks}) with self.roundtrip(data) as actual: - self.assertEqual(chunks, actual['var2'].encoding['chunks']) + assert chunks == actual['var2'].encoding['chunks'] # expect an error with non-integer chunks data['var2'].encoding.update({'chunks': (5, 4.5)}) @@ -1382,7 +1379,7 @@ def test_chunk_encoding_with_dask(self): # zarr automatically gets chunk information from dask chunks ds_chunk4 = ds.chunk({'x': 4}) with self.roundtrip(ds_chunk4) as actual: - self.assertEqual((4,), actual['var1'].encoding['chunks']) + assert (4,) == actual['var1'].encoding['chunks'] # should fail if dask_chunks are irregular... ds_chunk_irreg = ds.chunk({'x': (5, 4, 3)}) @@ -1395,14 +1392,14 @@ def test_chunk_encoding_with_dask(self): # ... except if the last chunk is smaller than the first ds_chunk_irreg = ds.chunk({'x': (5, 5, 2)}) with self.roundtrip(ds_chunk_irreg) as actual: - self.assertEqual((5,), actual['var1'].encoding['chunks']) + assert (5,) == actual['var1'].encoding['chunks'] # - encoding specified - # specify compatible encodings for chunk_enc in 4, (4, ): ds_chunk4['var1'].encoding.update({'chunks': chunk_enc}) with self.roundtrip(ds_chunk4) as actual: - self.assertEqual((4,), actual['var1'].encoding['chunks']) + assert (4,) == actual['var1'].encoding['chunks'] # TODO: remove this failure once syncronized overlapping writes are # supported by xarray @@ -1532,14 +1529,14 @@ def test_encoding_chunksizes(self): @requires_zarr -class ZarrDictStoreTest(BaseZarrTest, TestCase): +class ZarrDictStoreTest(BaseZarrTest): @contextlib.contextmanager def create_zarr_target(self): yield {} @requires_zarr -class ZarrDirectoryStoreTest(BaseZarrTest, TestCase): +class ZarrDirectoryStoreTest(BaseZarrTest): @contextlib.contextmanager def create_zarr_target(self): with create_tmp_file(suffix='.zarr') as tmp: @@ -1562,7 +1559,7 @@ def test_append_overwrite_values(self): @requires_scipy -class ScipyInMemoryDataTest(ScipyWriteTest, TestCase): +class ScipyInMemoryDataTest(ScipyWriteTest): engine = 'scipy' @contextlib.contextmanager @@ -1588,7 +1585,7 @@ class ScipyInMemoryDataTestAutocloseTrue(ScipyInMemoryDataTest): @requires_scipy -class ScipyFileObjectTest(ScipyWriteTest, TestCase): +class ScipyFileObjectTest(ScipyWriteTest): engine = 'scipy' @contextlib.contextmanager @@ -1616,7 +1613,7 @@ def test_pickle_dataarray(self): @requires_scipy -class ScipyFilePathTest(ScipyWriteTest, TestCase): +class ScipyFilePathTest(ScipyWriteTest): engine = 'scipy' @contextlib.contextmanager @@ -1640,7 +1637,7 @@ def test_netcdf3_endianness(self): # regression test for GH416 expected = open_example_dataset('bears.nc', engine='scipy') for var in expected.variables.values(): - self.assertTrue(var.dtype.isnative) + assert var.dtype.isnative @requires_netCDF4 def test_nc4_scipy(self): @@ -1657,7 +1654,7 @@ class ScipyFilePathTestAutocloseTrue(ScipyFilePathTest): @requires_netCDF4 -class NetCDF3ViaNetCDF4DataTest(CFEncodedDataTest, NetCDF3Only, TestCase): +class NetCDF3ViaNetCDF4DataTest(CFEncodedDataTest, NetCDF3Only): engine = 'netcdf4' file_format = 'NETCDF3_CLASSIC' @@ -1682,7 +1679,7 @@ class NetCDF3ViaNetCDF4DataTestAutocloseTrue(NetCDF3ViaNetCDF4DataTest): @requires_netCDF4 class NetCDF4ClassicViaNetCDF4DataTest(CFEncodedDataTest, NetCDF3Only, - TestCase): + object): engine = 'netcdf4' file_format = 'NETCDF4_CLASSIC' @@ -1700,7 +1697,7 @@ class NetCDF4ClassicViaNetCDF4DataTestAutocloseTrue( @requires_scipy_or_netCDF4 -class GenericNetCDFDataTest(CFEncodedDataTest, NetCDF3Only, TestCase): +class GenericNetCDFDataTest(CFEncodedDataTest, NetCDF3Only): # verify that we can read and write netCDF3 files as long as we have scipy # or netCDF4-python installed file_format = 'netcdf3_64bit' @@ -1754,24 +1751,24 @@ def test_encoding_unlimited_dims(self): ds = Dataset({'x': ('y', np.arange(10.0))}) with self.roundtrip(ds, save_kwargs=dict(unlimited_dims=['y'])) as actual: - self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert actual.encoding['unlimited_dims'] == set('y') assert_equal(ds, actual) # Regression test for https://github.com/pydata/xarray/issues/2134 with self.roundtrip(ds, save_kwargs=dict(unlimited_dims='y')) as actual: - self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert actual.encoding['unlimited_dims'] == set('y') assert_equal(ds, actual) ds.encoding = {'unlimited_dims': ['y']} with self.roundtrip(ds) as actual: - self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert actual.encoding['unlimited_dims'] == set('y') assert_equal(ds, actual) # Regression test for https://github.com/pydata/xarray/issues/2134 ds.encoding = {'unlimited_dims': 'y'} with self.roundtrip(ds) as actual: - self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert actual.encoding['unlimited_dims'] == set('y') assert_equal(ds, actual) @@ -1781,7 +1778,7 @@ class GenericNetCDFDataTestAutocloseTrue(GenericNetCDFDataTest): @requires_h5netcdf @requires_netCDF4 -class H5NetCDFDataTest(BaseNetCDF4Test, TestCase): +class H5NetCDFDataTest(BaseNetCDF4Test): engine = 'h5netcdf' @contextlib.contextmanager @@ -1822,11 +1819,11 @@ def test_encoding_unlimited_dims(self): ds = Dataset({'x': ('y', np.arange(10.0))}) with self.roundtrip(ds, save_kwargs=dict(unlimited_dims=['y'])) as actual: - self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert actual.encoding['unlimited_dims'] == set('y') assert_equal(ds, actual) ds.encoding = {'unlimited_dims': ['y']} with self.roundtrip(ds) as actual: - self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert actual.encoding['unlimited_dims'] == set('y') assert_equal(ds, actual) def test_compression_encoding_h5py(self): @@ -1857,7 +1854,7 @@ def test_compression_encoding_h5py(self): compr_out.update(compr_common) with self.roundtrip(data) as actual: for k, v in compr_out.items(): - self.assertEqual(v, actual['var2'].encoding[k]) + assert v == actual['var2'].encoding[k] def test_compression_check_encoding_h5py(self): """When mismatched h5py and NetCDF4-Python encodings are expressed @@ -1898,14 +1895,14 @@ def test_dump_encodings_h5py(self): kwargs = {'encoding': {'x': { 'compression': 'gzip', 'compression_opts': 9}}} with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertEqual(actual.x.encoding['zlib'], True) - self.assertEqual(actual.x.encoding['complevel'], 9) + assert actual.x.encoding['zlib'] + assert actual.x.encoding['complevel'] == 9 kwargs = {'encoding': {'x': { 'compression': 'lzf', 'compression_opts': None}}} with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertEqual(actual.x.encoding['compression'], 'lzf') - self.assertEqual(actual.x.encoding['compression_opts'], None) + assert actual.x.encoding['compression'] == 'lzf' + assert actual.x.encoding['compression_opts'] is None # tests pending h5netcdf fix @@ -1985,7 +1982,7 @@ def test_open_mfdataset_manyfiles(readengine, nfiles, autoclose, parallel, @requires_scipy_or_netCDF4 -class OpenMFDatasetWithDataVarsAndCoordsKwTest(TestCase): +class OpenMFDatasetWithDataVarsAndCoordsKwTest(object): coord_name = 'lon' var_name = 'v1' @@ -2056,9 +2053,9 @@ def test_common_coord_when_datavars_all(self): var_shape = ds[self.var_name].shape - self.assertEqual(var_shape, coord_shape) - self.assertNotEqual(coord_shape1, coord_shape) - self.assertNotEqual(coord_shape2, coord_shape) + assert var_shape == coord_shape + assert coord_shape1 != coord_shape + assert coord_shape2 != coord_shape def test_common_coord_when_datavars_minimal(self): opt = 'minimal' @@ -2073,9 +2070,9 @@ def test_common_coord_when_datavars_minimal(self): var_shape = ds[self.var_name].shape - self.assertNotEqual(var_shape, coord_shape) - self.assertEqual(coord_shape1, coord_shape) - self.assertEqual(coord_shape2, coord_shape) + assert var_shape != coord_shape + assert coord_shape1 == coord_shape + assert coord_shape2 == coord_shape def test_invalid_data_vars_value_should_fail(self): @@ -2093,7 +2090,7 @@ def test_invalid_data_vars_value_should_fail(self): @requires_dask @requires_scipy @requires_netCDF4 -class DaskTest(TestCase, DatasetIOTestCases): +class DaskTest(DatasetIOTestCases): @contextlib.contextmanager def create_store(self): yield Dataset() @@ -2133,10 +2130,10 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): with xr.set_options(enable_cftimeindex=True): with self.roundtrip(expected) as actual: abs_diff = abs(actual.t.values - expected_decoded_t) - self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + assert (abs_diff <= np.timedelta64(1, 's')).all() abs_diff = abs(actual.t0.values - expected_decoded_t0) - self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + assert (abs_diff <= np.timedelta64(1, 's')).all() def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): # Override method in DatasetIOTestCases - remove not applicable @@ -2153,10 +2150,10 @@ def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): with xr.set_options(enable_cftimeindex=False): with self.roundtrip(expected) as actual: abs_diff = abs(actual.t.values - expected_decoded_t) - self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + assert (abs_diff <= np.timedelta64(1, 's')).all() abs_diff = abs(actual.t0.values - expected_decoded_t0) - self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + assert (abs_diff <= np.timedelta64(1, 's')).all() def test_write_store(self): # Override method in DatasetIOTestCases - not applicable to dask @@ -2177,14 +2174,14 @@ def test_open_mfdataset(self): original.isel(x=slice(5, 10)).to_netcdf(tmp2) with open_mfdataset([tmp1, tmp2], autoclose=self.autoclose) as actual: - self.assertIsInstance(actual.foo.variable.data, da.Array) - self.assertEqual(actual.foo.variable.data.chunks, - ((5, 5),)) + assert isinstance(actual.foo.variable.data, da.Array) + assert actual.foo.variable.data.chunks == \ + ((5, 5),) assert_identical(original, actual) with open_mfdataset([tmp1, tmp2], chunks={'x': 3}, autoclose=self.autoclose) as actual: - self.assertEqual(actual.foo.variable.data.chunks, - ((3, 2, 3, 2),)) + assert actual.foo.variable.data.chunks == \ + ((3, 2, 3, 2),) with raises_regex(IOError, 'no files to open'): open_mfdataset('foo-bar-baz-*.nc', autoclose=self.autoclose) @@ -2218,7 +2215,7 @@ def test_attrs_mfdataset(self): with open_mfdataset([tmp1, tmp2]) as actual: # presumes that attributes inherited from # first dataset loaded - self.assertEqual(actual.test1, ds1.test1) + assert actual.test1 == ds1.test1 # attributes from ds2 are not retained, e.g., with raises_regex(AttributeError, 'no attribute'): @@ -2298,13 +2295,13 @@ def test_open_dataset(self): with create_tmp_file() as tmp: original.to_netcdf(tmp) with open_dataset(tmp, chunks={'x': 5}) as actual: - self.assertIsInstance(actual.foo.variable.data, da.Array) - self.assertEqual(actual.foo.variable.data.chunks, ((5, 5),)) + assert isinstance(actual.foo.variable.data, da.Array) + assert actual.foo.variable.data.chunks == ((5, 5),) assert_identical(original, actual) with open_dataset(tmp, chunks=5) as actual: assert_identical(original, actual) with open_dataset(tmp) as actual: - self.assertIsInstance(actual.foo.variable.data, np.ndarray) + assert isinstance(actual.foo.variable.data, np.ndarray) assert_identical(original, actual) def test_open_single_dataset(self): @@ -2344,9 +2341,9 @@ def test_deterministic_names(self): repeat_names = dict((k, v.data.name) for k, v in ds.data_vars.items()) for var_name, dask_name in original_names.items(): - self.assertIn(var_name, dask_name) - self.assertEqual(dask_name[:13], 'open_dataset-') - self.assertEqual(original_names, repeat_names) + assert var_name in dask_name + assert dask_name[:13] == 'open_dataset-' + assert original_names == repeat_names def test_dataarray_compute(self): # Test DataArray.compute() on dask backend. @@ -2354,8 +2351,8 @@ def test_dataarray_compute(self): # however dask is the only tested backend which supports DataArrays actual = DataArray([1, 2]).chunk() computed = actual.compute() - self.assertFalse(actual._in_memory) - self.assertTrue(computed._in_memory) + assert not actual._in_memory + assert computed._in_memory assert_allclose(actual, computed, decode_bytes=False) def test_to_netcdf_compute_false_roundtrip(self): @@ -2395,7 +2392,7 @@ class DaskTestAutocloseTrue(DaskTest): @requires_scipy_or_netCDF4 @requires_pydap -class PydapTest(TestCase): +class PydapTest(object): def convert_to_pydap_dataset(self, original): from pydap.model import GridType, BaseType, DatasetType ds = DatasetType('bears', **original.attrs) @@ -2427,8 +2424,8 @@ def test_cmp_local_file(self): assert_equal(actual, expected) # global attributes should be global attributes on the dataset - self.assertNotIn('NC_GLOBAL', actual.attrs) - self.assertIn('history', actual.attrs) + assert 'NC_GLOBAL' not in actual.attrs + assert 'history' in actual.attrs # we don't check attributes exactly with assertDatasetIdentical() # because the test DAP server seems to insert some extra @@ -2436,8 +2433,7 @@ def test_cmp_local_file(self): assert actual.attrs.keys() == expected.attrs.keys() with self.create_datasets() as (actual, expected): - assert_equal( - actual.isel(l=2), expected.isel(l=2)) # noqa: E741 + assert_equal(actual.isel(l=2), expected.isel(l=2)) # noqa with self.create_datasets() as (actual, expected): assert_equal(actual.isel(i=0, j=-1), @@ -2497,7 +2493,7 @@ def test_session(self): @requires_scipy @requires_pynio -class PyNioTest(ScipyWriteTest, TestCase): +class PyNioTest(ScipyWriteTest): def test_write_store(self): # pynio is read-only for now pass @@ -2529,7 +2525,7 @@ class PyNioTestAutocloseTrue(PyNioTest): @requires_pseudonetcdf @pytest.mark.filterwarnings('ignore:IOAPI_ISPH is assumed to be 6370000') -class PseudoNetCDFFormatTest(TestCase): +class PseudoNetCDFFormatTest(object): autoclose = True def open(self, path, **kwargs): @@ -2792,7 +2788,7 @@ def create_tmp_geotiff(nx=4, ny=3, nz=3, @requires_rasterio -class TestRasterio(TestCase): +class TestRasterio(object): @requires_scipy_or_netCDF4 def test_serialization(self): @@ -2837,7 +2833,8 @@ def test_non_rectilinear(self): assert len(rioda.attrs['transform']) == 6 # See if a warning is raised if we force it - with self.assertWarns("transformation isn't rectilinear"): + with pytest.warns(Warning, + match="transformation isn't rectilinear"): with xr.open_rasterio(tmp_file, parse_coordinates=True) as rioda: assert 'x' not in rioda.coords @@ -3024,7 +3021,7 @@ def test_chunks(self): with xr.open_rasterio(tmp_file, chunks=(1, 2, 2)) as actual: import dask.array as da - self.assertIsInstance(actual.data, da.Array) + assert isinstance(actual.data, da.Array) assert 'open_rasterio' in actual.data.name # do some arithmetic @@ -3105,7 +3102,7 @@ def test_no_mftime(self): with mock.patch('os.path.getmtime', side_effect=OSError): with xr.open_rasterio(tmp_file, chunks=(1, 2, 2)) as actual: import dask.array as da - self.assertIsInstance(actual.data, da.Array) + assert isinstance(actual.data, da.Array) assert_allclose(actual, expected) @network @@ -3118,10 +3115,10 @@ def test_http_url(self): # make sure chunking works with xr.open_rasterio(url, chunks=(1, 256, 256)) as actual: import dask.array as da - self.assertIsInstance(actual.data, da.Array) + assert isinstance(actual.data, da.Array) -class TestEncodingInvalid(TestCase): +class TestEncodingInvalid(object): def test_extract_nc4_variable_encoding(self): var = xr.Variable(('x',), [1, 2, 3], {}, {'foo': 'bar'}) @@ -3130,12 +3127,12 @@ def test_extract_nc4_variable_encoding(self): var = xr.Variable(('x',), [1, 2, 3], {}, {'chunking': (2, 1)}) encoding = _extract_nc4_variable_encoding(var) - self.assertEqual({}, encoding) + assert {} == encoding # regression test var = xr.Variable(('x',), [1, 2, 3], {}, {'shuffle': True}) encoding = _extract_nc4_variable_encoding(var, raise_on_invalid=True) - self.assertEqual({'shuffle': True}, encoding) + assert {'shuffle': True} == encoding def test_extract_h5nc_encoding(self): # not supported with h5netcdf (yet) @@ -3150,7 +3147,7 @@ class MiscObject: @requires_netCDF4 -class TestValidateAttrs(TestCase): +class TestValidateAttrs(object): def test_validating_attrs(self): def new_dataset(): return Dataset({'data': ('y', np.arange(10.0))}, @@ -3250,7 +3247,7 @@ def new_dataset_and_coord_attrs(): @requires_scipy_or_netCDF4 -class TestDataArrayToNetCDF(TestCase): +class TestDataArrayToNetCDF(object): def test_dataarray_to_netcdf_no_name(self): original_da = DataArray(np.arange(12).reshape((3, 4))) diff --git a/xarray/tests/test_combine.py b/xarray/tests/test_combine.py index 482a280b355..2004b1e660f 100644 --- a/xarray/tests/test_combine.py +++ b/xarray/tests/test_combine.py @@ -10,12 +10,12 @@ from xarray.core.pycompat import OrderedDict, iteritems from . import ( - InaccessibleArray, TestCase, assert_array_equal, assert_equal, - assert_identical, raises_regex, requires_dask) + InaccessibleArray, assert_array_equal, assert_equal, assert_identical, + raises_regex, requires_dask) from .test_dataset import create_test_data -class TestConcatDataset(TestCase): +class TestConcatDataset(object): def test_concat(self): # TODO: simplify and split this test case @@ -235,7 +235,7 @@ def test_concat_multiindex(self): assert isinstance(actual.x.to_index(), pd.MultiIndex) -class TestConcatDataArray(TestCase): +class TestConcatDataArray(object): def test_concat(self): ds = Dataset({'foo': (['x', 'y'], np.random.random((2, 3))), 'bar': (['x', 'y'], np.random.random((2, 3)))}, @@ -295,7 +295,7 @@ def test_concat_lazy(self): assert combined.dims == ('z', 'x', 'y') -class TestAutoCombine(TestCase): +class TestAutoCombine(object): @requires_dask # only for toolz def test_auto_combine(self): diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 5ed482ed2bd..a067d01a308 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -8,20 +8,20 @@ import pandas as pd import pytest -from xarray import (Dataset, Variable, SerializationWarning, coding, - conventions, open_dataset) +from xarray import ( + Dataset, SerializationWarning, Variable, coding, conventions, open_dataset) from xarray.backends.common import WritableCFDataStore from xarray.backends.memory import InMemoryDataStore from xarray.conventions import decode_cf from xarray.testing import assert_identical from . import ( - TestCase, assert_array_equal, raises_regex, requires_netCDF4, - requires_cftime_or_netCDF4, unittest, requires_dask) + assert_array_equal, raises_regex, requires_cftime_or_netCDF4, + requires_dask, requires_netCDF4) from .test_backends import CFEncodedDataTest -class TestBoolTypeArray(TestCase): +class TestBoolTypeArray(object): def test_booltype_array(self): x = np.array([1, 0, 1, 1, 0], dtype='i1') bx = conventions.BoolTypeArray(x) @@ -30,7 +30,7 @@ def test_booltype_array(self): dtype=np.bool)) -class TestNativeEndiannessArray(TestCase): +class TestNativeEndiannessArray(object): def test(self): x = np.arange(5, dtype='>i8') expected = np.arange(5, dtype='int64') @@ -69,7 +69,7 @@ def test_decode_cf_with_conflicting_fill_missing_value(): @requires_cftime_or_netCDF4 -class TestEncodeCFVariable(TestCase): +class TestEncodeCFVariable(object): def test_incompatible_attributes(self): invalid_vars = [ Variable(['t'], pd.date_range('2000-01-01', periods=3), @@ -134,7 +134,7 @@ def test_string_object_warning(self): @requires_cftime_or_netCDF4 -class TestDecodeCF(TestCase): +class TestDecodeCF(object): def test_dataset(self): original = Dataset({ 't': ('t', [0, 1, 2], {'units': 'days since 2000-01-01'}), @@ -255,7 +255,7 @@ def encode_variable(self, var): @requires_netCDF4 -class TestCFEncodedDataStore(CFEncodedDataTest, TestCase): +class TestCFEncodedDataStore(CFEncodedDataTest): @contextlib.contextmanager def create_store(self): yield CFEncodedInMemoryStore() @@ -267,9 +267,10 @@ def roundtrip(self, data, save_kwargs={}, open_kwargs={}, data.dump_to_store(store, **save_kwargs) yield open_dataset(store, **open_kwargs) + @pytest.mark.skip('cannot roundtrip coordinates yet for ' + 'CFEncodedInMemoryStore') def test_roundtrip_coordinates(self): - raise unittest.SkipTest('cannot roundtrip coordinates yet for ' - 'CFEncodedInMemoryStore') + pass def test_invalid_dataarray_names_raise(self): # only relevant for on-disk file formats diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 43fa35473ce..e56f751bef9 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, division, print_function import pickle -from textwrap import dedent from distutils.version import LooseVersion +from textwrap import dedent import numpy as np import pandas as pd @@ -15,15 +15,15 @@ from xarray.tests import mock from . import ( - TestCase, assert_allclose, assert_array_equal, assert_equal, - assert_frame_equal, assert_identical, raises_regex) + assert_allclose, assert_array_equal, assert_equal, assert_frame_equal, + assert_identical, raises_regex) dask = pytest.importorskip('dask') da = pytest.importorskip('dask.array') dd = pytest.importorskip('dask.dataframe') -class DaskTestCase(TestCase): +class DaskTestCase(object): def assertLazyAnd(self, expected, actual, test): with (dask.config.set(get=dask.get) @@ -57,6 +57,7 @@ def assertLazyAndIdentical(self, expected, actual): def assertLazyAndAllClose(self, expected, actual): self.assertLazyAnd(expected, actual, assert_allclose) + @pytest.fixture(autouse=True) def setUp(self): self.values = np.random.RandomState(0).randn(4, 6) self.data = da.from_array(self.values, chunks=(2, 2)) @@ -249,6 +250,7 @@ def assertLazyAndAllClose(self, expected, actual): def assertLazyAndEqual(self, expected, actual): self.assertLazyAnd(expected, actual, assert_equal) + @pytest.fixture(autouse=True) def setUp(self): self.values = np.random.randn(4, 6) self.data = da.from_array(self.values, chunks=(2, 2)) @@ -581,7 +583,7 @@ def test_from_dask_variable(self): self.assertLazyAndIdentical(self.lazy_array, a) -class TestToDaskDataFrame(TestCase): +class TestToDaskDataFrame(object): def test_to_dask_dataframe(self): # Test conversion of Datasets to dask DataFrames diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index f8b288f4ab0..d15a0bb6081 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1,10 +1,9 @@ from __future__ import absolute_import, division, print_function import pickle +import warnings from copy import deepcopy -from distutils.version import LooseVersion from textwrap import dedent -import warnings import numpy as np import pandas as pd @@ -13,19 +12,20 @@ import xarray as xr from xarray import ( DataArray, Dataset, IndexVariable, Variable, align, broadcast, set_options) -from xarray.convert import from_cdms2 from xarray.coding.times import CFDatetimeCoder, _import_cftime -from xarray.core.common import full_like, ALL_DIMS +from xarray.convert import from_cdms2 +from xarray.core.common import ALL_DIMS, full_like from xarray.core.pycompat import OrderedDict, iteritems from xarray.tests import ( - ReturnItem, TestCase, assert_allclose, assert_array_equal, assert_equal, + ReturnItem, assert_allclose, assert_array_equal, assert_equal, assert_identical, raises_regex, requires_bottleneck, requires_cftime, requires_dask, requires_iris, requires_np113, requires_scipy, - source_ndarray, unittest) + source_ndarray) -class TestDataArray(TestCase): - def setUp(self): +class TestDataArray(object): + @pytest.fixture(autouse=True) + def setup(self): self.attrs = {'attr1': 'value1', 'attr2': 2929} self.x = np.random.random((10, 20)) self.v = Variable(['x', 'y'], self.x) @@ -441,7 +441,7 @@ def test_getitem(self): assert_identical(self.ds['x'], x) assert_identical(self.ds['y'], y) - I = ReturnItem() # noqa: E741 # allow ambiguous name + I = ReturnItem() # noqa for i in [I[:], I[...], I[x.values], I[x.variable], I[x], I[x, y], I[x.values > -1], I[x.variable > -1], I[x > -1], I[x > -1, y > -1]]: @@ -1002,7 +1002,7 @@ def test_sel(lab_indexer, pos_indexer, replaced_idx=False, assert da.dims[0] == renamed_dim da = da.rename({renamed_dim: 'x'}) assert_identical(da.variable, expected_da.variable) - self.assertVariableNotEqual(da['x'], expected_da['x']) + assert not da['x'].equals(expected_da['x']) test_sel(('a', 1, -1), 0) test_sel(('b', 2, -2), -1) @@ -2026,17 +2026,19 @@ def test_groupby_warning(self): with pytest.warns(FutureWarning): grouped.sum() - @pytest.mark.skipif(LooseVersion(xr.__version__) < LooseVersion('0.12'), - reason="not to forget the behavior change") + # Currently disabled due to https://github.com/pydata/xarray/issues/2468 + # @pytest.mark.skipif(LooseVersion(xr.__version__) < LooseVersion('0.12'), + # reason="not to forget the behavior change") + @pytest.mark.skip def test_groupby_sum_default(self): array = self.make_groupby_example_array() grouped = array.groupby('abc') expected_sum_all = Dataset( {'foo': Variable(['x', 'abc'], - np.array([self.x[:, :9].sum(axis=-1), - self.x[:, 10:].sum(axis=-1), - self.x[:, 9:10].sum(axis=-1)]).T), + np.array([self.x[:, :9].sum(axis=-1), + self.x[:, 10:].sum(axis=-1), + self.x[:, 9:10].sum(axis=-1)]).T), 'abc': Variable(['abc'], np.array(['a', 'b', 'c']))})['foo'] assert_allclose(expected_sum_all, grouped.sum()) @@ -2050,7 +2052,7 @@ def test_groupby_count(self): expected = DataArray([1, 1, 2], coords=[('cat', ['a', 'b', 'c'])]) assert_identical(actual, expected) - @unittest.skip('needs to be fixed for shortcut=False, keep_attrs=False') + @pytest.mark.skip('needs to be fixed for shortcut=False, keep_attrs=False') def test_groupby_reduce_attrs(self): array = self.make_groupby_example_array() array.attrs['foo'] = 'bar' @@ -2826,7 +2828,7 @@ def test_to_and_from_series(self): def test_series_categorical_index(self): # regression test for GH700 if not hasattr(pd, 'CategoricalIndex'): - raise unittest.SkipTest('requires pandas with CategoricalIndex') + pytest.skip('requires pandas with CategoricalIndex') s = pd.Series(np.arange(5), index=pd.CategoricalIndex(list('aabbc'))) arr = DataArray(s) @@ -2968,7 +2970,7 @@ def test_to_and_from_cdms2_classic(self): actual = original.to_cdms2() assert_array_equal(actual.asma(), original) assert actual.id == original.name - self.assertItemsEqual(actual.getAxisIds(), original.dims) + assert tuple(actual.getAxisIds()) == original.dims for axis, coord in zip(actual.getAxisList(), expected_coords): assert axis.id == coord.name assert_array_equal(axis, coord.values) @@ -2982,8 +2984,8 @@ def test_to_and_from_cdms2_classic(self): assert_identical(original, roundtripped) back = from_cdms2(actual) - self.assertItemsEqual(original.dims, back.dims) - self.assertItemsEqual(original.coords.keys(), back.coords.keys()) + assert original.dims == back.dims + assert original.coords.keys() == back.coords.keys() for coord_name in original.coords.keys(): assert_array_equal(original.coords[coord_name], back.coords[coord_name]) @@ -3004,15 +3006,15 @@ def test_to_and_from_cdms2_sgrid(self): coords=OrderedDict(x=x, y=y, lon=lon, lat=lat), name='sst') actual = original.to_cdms2() - self.assertItemsEqual(actual.getAxisIds(), original.dims) + assert tuple(actual.getAxisIds()) == original.dims assert_array_equal(original.coords['lon'], actual.getLongitude().asma()) assert_array_equal(original.coords['lat'], actual.getLatitude().asma()) back = from_cdms2(actual) - self.assertItemsEqual(original.dims, back.dims) - self.assertItemsEqual(original.coords.keys(), back.coords.keys()) + assert original.dims == back.dims + assert set(original.coords.keys()) == set(back.coords.keys()) assert_array_equal(original.coords['lat'], back.coords['lat']) assert_array_equal(original.coords['lon'], back.coords['lon']) @@ -3026,15 +3028,15 @@ def test_to_and_from_cdms2_ugrid(self): original = DataArray(np.arange(5), dims=['cell'], coords={'lon': lon, 'lat': lat, 'cell': cell}) actual = original.to_cdms2() - self.assertItemsEqual(actual.getAxisIds(), original.dims) + assert tuple(actual.getAxisIds()) == original.dims assert_array_equal(original.coords['lon'], actual.getLongitude().getValue()) assert_array_equal(original.coords['lat'], actual.getLatitude().getValue()) back = from_cdms2(actual) - self.assertItemsEqual(original.dims, back.dims) - self.assertItemsEqual(original.coords.keys(), back.coords.keys()) + assert set(original.dims) == set(back.dims) + assert set(original.coords.keys()) == set(back.coords.keys()) assert_array_equal(original.coords['lat'], back.coords['lat']) assert_array_equal(original.coords['lon'], back.coords['lon']) @@ -3127,17 +3129,17 @@ def test_coordinate_diff(self): actual = lon.diff('lon') assert_equal(expected, actual) - def test_shift(self): + @pytest.mark.parametrize('offset', [-5, -2, -1, 0, 1, 2, 5]) + def test_shift(self, offset): arr = DataArray([1, 2, 3], dims='x') actual = arr.shift(x=1) expected = DataArray([np.nan, 1, 2], dims='x') assert_identical(expected, actual) arr = DataArray([1, 2, 3], [('x', ['a', 'b', 'c'])]) - for offset in [-5, -2, -1, 0, 1, 2, 5]: - expected = DataArray(arr.to_pandas().shift(offset)) - actual = arr.shift(x=offset) - assert_identical(expected, actual) + expected = DataArray(arr.to_pandas().shift(offset)) + actual = arr.shift(x=offset) + assert_identical(expected, actual) def test_roll_coords(self): arr = DataArray([1, 2, 3], coords={'x': range(3)}, dims='x') diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 2c964b81b98..9bee965392b 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function +import sys +import warnings from copy import copy, deepcopy from io import StringIO from textwrap import dedent -import warnings -import sys import numpy as np import pandas as pd @@ -13,17 +13,17 @@ import xarray as xr from xarray import ( - DataArray, Dataset, IndexVariable, MergeError, Variable, align, backends, - broadcast, open_dataset, set_options, ALL_DIMS) + ALL_DIMS, DataArray, Dataset, IndexVariable, MergeError, Variable, align, + backends, broadcast, open_dataset, set_options) from xarray.core import indexing, npcompat, utils from xarray.core.common import full_like from xarray.core.pycompat import ( OrderedDict, integer_types, iteritems, unicode_type) from . import ( - InaccessibleArray, TestCase, UnexpectedDataAccess, assert_allclose, - assert_array_equal, assert_equal, assert_identical, has_cftime, - has_dask, raises_regex, requires_bottleneck, requires_dask, requires_scipy, + InaccessibleArray, UnexpectedDataAccess, assert_allclose, + assert_array_equal, assert_equal, assert_identical, has_cftime, has_dask, + raises_regex, requires_bottleneck, requires_dask, requires_scipy, source_ndarray) try: @@ -86,7 +86,7 @@ def lazy_inaccessible(k, v): k, v in iteritems(self._variables)) -class TestDataset(TestCase): +class TestDataset(object): def test_repr(self): data = create_test_data(seed=123) data.attrs['foo'] = 'bar' @@ -399,7 +399,7 @@ def test_constructor_with_coords(self): ds = Dataset({}, {'a': ('x', [1])}) assert not ds.data_vars - self.assertItemsEqual(ds.coords.keys(), ['a']) + assert list(ds.coords.keys()) == ['a'] mindex = pd.MultiIndex.from_product([['a', 'b'], [1, 2]], names=('level_1', 'level_2')) @@ -421,9 +421,9 @@ def test_properties(self): assert type(ds.dims.mapping.mapping) is dict # noqa with pytest.warns(FutureWarning): - self.assertItemsEqual(ds, list(ds.variables)) + assert list(ds) == list(ds.variables) with pytest.warns(FutureWarning): - self.assertItemsEqual(ds.keys(), list(ds.variables)) + assert list(ds.keys()) == list(ds.variables) assert 'aasldfjalskdfj' not in ds.variables assert 'dim1' in repr(ds.variables) with pytest.warns(FutureWarning): @@ -431,18 +431,18 @@ def test_properties(self): with pytest.warns(FutureWarning): assert bool(ds) - self.assertItemsEqual(ds.data_vars, ['var1', 'var2', 'var3']) - self.assertItemsEqual(ds.data_vars.keys(), ['var1', 'var2', 'var3']) + assert list(ds.data_vars) == ['var1', 'var2', 'var3'] + assert list(ds.data_vars.keys()) == ['var1', 'var2', 'var3'] assert 'var1' in ds.data_vars assert 'dim1' not in ds.data_vars assert 'numbers' not in ds.data_vars assert len(ds.data_vars) == 3 - self.assertItemsEqual(ds.indexes, ['dim2', 'dim3', 'time']) + assert set(ds.indexes) == {'dim2', 'dim3', 'time'} assert len(ds.indexes) == 3 assert 'dim2' in repr(ds.indexes) - self.assertItemsEqual(ds.coords, ['time', 'dim2', 'dim3', 'numbers']) + assert list(ds.coords) == ['time', 'dim2', 'dim3', 'numbers'] assert 'dim2' in ds.coords assert 'numbers' in ds.coords assert 'var1' not in ds.coords @@ -535,7 +535,7 @@ def test_coords_properties(self): assert 4 == len(data.coords) - self.assertItemsEqual(['x', 'y', 'a', 'b'], list(data.coords)) + assert ['x', 'y', 'a', 'b'] == list(data.coords) assert_identical(data.coords['x'].variable, data['x'].variable) assert_identical(data.coords['y'].variable, data['y'].variable) @@ -831,7 +831,7 @@ def test_isel(self): ret = data.isel(**slicers) # Verify that only the specified dimension was altered - self.assertItemsEqual(data.dims, ret.dims) + assert list(data.dims) == list(ret.dims) for d in data.dims: if d in slicers: assert ret.dims[d] == \ @@ -857,21 +857,21 @@ def test_isel(self): ret = data.isel(dim1=0) assert {'time': 20, 'dim2': 9, 'dim3': 10} == ret.dims - self.assertItemsEqual(data.data_vars, ret.data_vars) - self.assertItemsEqual(data.coords, ret.coords) - self.assertItemsEqual(data.indexes, ret.indexes) + assert set(data.data_vars) == set(ret.data_vars) + assert set(data.coords) == set(ret.coords) + assert set(data.indexes) == set(ret.indexes) ret = data.isel(time=slice(2), dim1=0, dim2=slice(5)) assert {'time': 2, 'dim2': 5, 'dim3': 10} == ret.dims - self.assertItemsEqual(data.data_vars, ret.data_vars) - self.assertItemsEqual(data.coords, ret.coords) - self.assertItemsEqual(data.indexes, ret.indexes) + assert set(data.data_vars) == set(ret.data_vars) + assert set(data.coords) == set(ret.coords) + assert set(data.indexes) == set(ret.indexes) ret = data.isel(time=0, dim1=0, dim2=slice(5)) - self.assertItemsEqual({'dim2': 5, 'dim3': 10}, ret.dims) - self.assertItemsEqual(data.data_vars, ret.data_vars) - self.assertItemsEqual(data.coords, ret.coords) - self.assertItemsEqual(data.indexes, list(ret.indexes) + ['time']) + assert {'dim2': 5, 'dim3': 10} == ret.dims + assert set(data.data_vars) == set(ret.data_vars) + assert set(data.coords) == set(ret.coords) + assert set(data.indexes) == set(list(ret.indexes) + ['time']) def test_isel_fancy(self): # isel with fancy indexing. @@ -1482,7 +1482,7 @@ def test_sel(lab_indexer, pos_indexer, replaced_idx=False, ds = ds.rename({renamed_dim: 'x'}) assert_identical(ds['var'].variable, expected_ds['var'].variable) - self.assertVariableNotEqual(ds['x'], expected_ds['x']) + assert not ds['x'].equals(expected_ds['x']) test_sel(('a', 1, -1), 0) test_sel(('b', 2, -2), -1) @@ -2546,12 +2546,11 @@ def test_setitem_multiindex_level(self): def test_delitem(self): data = create_test_data() all_items = set(data.variables) - self.assertItemsEqual(data.variables, all_items) + assert set(data.variables) == all_items del data['var1'] - self.assertItemsEqual(data.variables, all_items - set(['var1'])) + assert set(data.variables) == all_items - set(['var1']) del data['numbers'] - self.assertItemsEqual(data.variables, - all_items - set(['var1', 'numbers'])) + assert set(data.variables) == all_items - set(['var1', 'numbers']) assert 'numbers' not in data.coords def test_squeeze(self): @@ -3425,8 +3424,8 @@ def test_reduce(self): (['dim2', 'time'], ['dim1', 'dim3']), (('dim2', 'time'), ['dim1', 'dim3']), ((), ['dim1', 'dim2', 'dim3', 'time'])]: - actual = data.min(dim=reduct).dims - self.assertItemsEqual(actual, expected) + actual = list(data.min(dim=reduct).dims) + assert actual == expected assert_equal(data.mean(dim=[]), data) @@ -3480,7 +3479,7 @@ def test_reduce_cumsum_test_dims(self): ('time', ['dim1', 'dim2', 'dim3']) ]: actual = getattr(data, cumfunc)(dim=reduct).dims - self.assertItemsEqual(actual, expected) + assert list(actual) == expected def test_reduce_non_numeric(self): data1 = create_test_data(seed=44) @@ -3618,14 +3617,14 @@ def test_rank(self): ds = create_test_data(seed=1234) # only ds.var3 depends on dim3 z = ds.rank('dim3') - self.assertItemsEqual(['var3'], list(z.data_vars)) + assert ['var3'] == list(z.data_vars) # same as dataarray version x = z.var3 y = ds.var3.rank('dim3') assert_equal(x, y) # coordinates stick - self.assertItemsEqual(list(z.coords), list(ds.coords)) - self.assertItemsEqual(list(x.coords), list(y.coords)) + assert list(z.coords) == list(ds.coords) + assert list(x.coords) == list(y.coords) # invalid dim with raises_regex(ValueError, 'does not contain'): x.rank('invalid_dim') @@ -3948,10 +3947,10 @@ def test_roll_coords_none(self): def test_roll_multidim(self): # regression test for 2445 arr = xr.DataArray( - [[1, 2, 3],[4, 5, 6]], coords={'x': range(3), 'y': range(2)}, - dims=('y','x')) + [[1, 2, 3], [4, 5, 6]], coords={'x': range(3), 'y': range(2)}, + dims=('y', 'x')) actual = arr.roll(x=1, roll_coords=True) - expected = xr.DataArray([[3, 1, 2],[6, 4, 5]], + expected = xr.DataArray([[3, 1, 2], [6, 4, 5]], coords=[('y', [0, 1]), ('x', [2, 0, 1])]) assert_identical(expected, actual) diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index aab5d305a82..5ea5b3d2a42 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -1,16 +1,16 @@ from __future__ import absolute_import, division, print_function +import warnings from distutils.version import LooseVersion +from textwrap import dedent import numpy as np import pandas as pd import pytest -from textwrap import dedent from numpy import array, nan -import warnings from xarray import DataArray, Dataset, concat -from xarray.core import duck_array_ops, dtypes +from xarray.core import dtypes, duck_array_ops from xarray.core.duck_array_ops import ( array_notnull_equiv, concatenate, count, first, gradient, last, mean, rolling_window, stack, where) @@ -18,12 +18,12 @@ from xarray.testing import assert_allclose, assert_equal from . import ( - TestCase, assert_array_equal, has_dask, has_np113, raises_regex, - requires_dask) + assert_array_equal, has_dask, has_np113, raises_regex, requires_dask) -class TestOps(TestCase): +class TestOps(object): + @pytest.fixture(autouse=True) def setUp(self): self.x = array([[[nan, nan, 2., nan], [nan, 5., 6., nan], diff --git a/xarray/tests/test_extensions.py b/xarray/tests/test_extensions.py index 24b710ae223..ffefa78aa34 100644 --- a/xarray/tests/test_extensions.py +++ b/xarray/tests/test_extensions.py @@ -4,7 +4,7 @@ import xarray as xr -from . import TestCase, raises_regex +from . import raises_regex try: import cPickle as pickle @@ -21,7 +21,7 @@ def __init__(self, xarray_obj): self.obj = xarray_obj -class TestAccessor(TestCase): +class TestAccessor(object): def test_register(self): @xr.register_dataset_accessor('demo') diff --git a/xarray/tests/test_formatting.py b/xarray/tests/test_formatting.py index 8a1003f1ced..024c669bed9 100644 --- a/xarray/tests/test_formatting.py +++ b/xarray/tests/test_formatting.py @@ -7,10 +7,10 @@ from xarray.core import formatting from xarray.core.pycompat import PY3 -from . import TestCase, raises_regex +from . import raises_regex -class TestFormatting(TestCase): +class TestFormatting(object): def test_get_indexer_at_least_n_items(self): cases = [ @@ -45,7 +45,7 @@ def test_first_n_items(self): for n in [3, 10, 13, 100, 200]: actual = formatting.first_n_items(array, n) expected = array.flat[:n] - self.assertItemsEqual(expected, actual) + assert (expected == actual).all() with raises_regex(ValueError, 'at least one item'): formatting.first_n_items(array, 0) @@ -55,7 +55,7 @@ def test_last_n_items(self): for n in [3, 10, 13, 100, 200]: actual = formatting.last_n_items(array, n) expected = array.flat[-n:] - self.assertItemsEqual(expected, actual) + assert (expected == actual).all() with raises_regex(ValueError, 'at least one item'): formatting.first_n_items(array, 0) diff --git a/xarray/tests/test_indexing.py b/xarray/tests/test_indexing.py index 0d1045d35c0..701eefcb462 100644 --- a/xarray/tests/test_indexing.py +++ b/xarray/tests/test_indexing.py @@ -10,13 +10,12 @@ from xarray.core import indexing, nputils from xarray.core.pycompat import native_int_types -from . import ( - IndexerMaker, ReturnItem, TestCase, assert_array_equal, raises_regex) +from . import IndexerMaker, ReturnItem, assert_array_equal, raises_regex B = IndexerMaker(indexing.BasicIndexer) -class TestIndexers(TestCase): +class TestIndexers(object): def set_to_zero(self, x, i): x = x.copy() x[i] = 0 @@ -25,7 +24,7 @@ def set_to_zero(self, x, i): def test_expanded_indexer(self): x = np.random.randn(10, 11, 12, 13, 14) y = np.arange(5) - I = ReturnItem() # noqa: E741 # allow ambiguous name + I = ReturnItem() # noqa for i in [I[:], I[...], I[0, :, 10], I[..., 10], I[:5, ..., 0], I[..., 0, :], I[y], I[y, y], I[..., y, y], I[..., 0, 1, 2, 3, 4]]: @@ -133,7 +132,7 @@ def test_indexer(data, x, expected_pos, expected_idx=None): pd.MultiIndex.from_product([[1, 2], [-1, -2]])) -class TestLazyArray(TestCase): +class TestLazyArray(object): def test_slice_slice(self): I = ReturnItem() # noqa: E741 # allow ambiguous name for size in [100, 99]: @@ -248,7 +247,7 @@ def check_indexing(v_eager, v_lazy, indexers): check_indexing(v_eager, v_lazy, indexers) -class TestCopyOnWriteArray(TestCase): +class TestCopyOnWriteArray(object): def test_setitem(self): original = np.arange(10) wrapped = indexing.CopyOnWriteArray(original) @@ -272,7 +271,7 @@ def test_index_scalar(self): assert np.array(x[B[0]][B[()]]) == 'foo' -class TestMemoryCachedArray(TestCase): +class TestMemoryCachedArray(object): def test_wrapper(self): original = indexing.LazilyOuterIndexedArray(np.arange(10)) wrapped = indexing.MemoryCachedArray(original) @@ -385,8 +384,9 @@ def test_vectorized_indexer(): np.arange(5, dtype=np.int64))) -class Test_vectorized_indexer(TestCase): - def setUp(self): +class Test_vectorized_indexer(object): + @pytest.fixture(autouse=True) + def setup(self): self.data = indexing.NumpyIndexingAdapter(np.random.randn(10, 12, 13)) self.indexers = [np.array([[0, 3, 2], ]), np.array([[0, 3, 3], [4, 6, 7]]), diff --git a/xarray/tests/test_merge.py b/xarray/tests/test_merge.py index 4d89be8ce55..300c490cff6 100644 --- a/xarray/tests/test_merge.py +++ b/xarray/tests/test_merge.py @@ -6,11 +6,11 @@ import xarray as xr from xarray.core import merge -from . import TestCase, raises_regex +from . import raises_regex from .test_dataset import create_test_data -class TestMergeInternals(TestCase): +class TestMergeInternals(object): def test_broadcast_dimension_size(self): actual = merge.broadcast_dimension_size( [xr.Variable('x', [1]), xr.Variable('y', [2, 1])]) @@ -25,7 +25,7 @@ def test_broadcast_dimension_size(self): [xr.Variable(('x', 'y'), [[1, 2]]), xr.Variable('y', [2])]) -class TestMergeFunction(TestCase): +class TestMergeFunction(object): def test_merge_arrays(self): data = create_test_data() actual = xr.merge([data.var1, data.var2]) @@ -130,7 +130,7 @@ def test_merge_no_conflicts_broadcast(self): assert expected.identical(actual) -class TestMergeMethod(TestCase): +class TestMergeMethod(object): def test_merge(self): data = create_test_data() @@ -195,7 +195,7 @@ def test_merge_compat(self): with pytest.raises(xr.MergeError): ds1.merge(ds2, compat='identical') - with raises_regex(ValueError, 'compat=\S+ invalid'): + with raises_regex(ValueError, 'compat=.* invalid'): ds1.merge(ds2, compat='foobar') def test_merge_auto_align(self): diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 98265149122..01303202c93 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -5,9 +5,9 @@ import numpy as np import pandas as pd -import xarray as xr import pytest +import xarray as xr import xarray.plot as xplt from xarray import DataArray from xarray.coding.times import _import_cftime @@ -17,9 +17,8 @@ import_seaborn, label_from_attrs) from . import ( - TestCase, assert_array_equal, assert_equal, raises_regex, - requires_matplotlib, requires_matplotlib2, requires_seaborn, - requires_cftime) + assert_array_equal, assert_equal, raises_regex, requires_cftime, + requires_matplotlib, requires_matplotlib2, requires_seaborn) # import mpl and change the backend before other mpl imports try: @@ -65,8 +64,10 @@ def easy_array(shape, start=0, stop=1): @requires_matplotlib -class PlotTestCase(TestCase): - def tearDown(self): +class PlotTestCase(object): + @pytest.fixture(autouse=True) + def setup(self): + yield # Remove all matplotlib figures plt.close('all') @@ -88,7 +89,8 @@ def contourf_called(self, plotmethod): class TestPlot(PlotTestCase): - def setUp(self): + @pytest.fixture(autouse=True) + def setup_array(self): self.darray = DataArray(easy_array((2, 3, 4))) def test_label_from_attrs(self): @@ -160,8 +162,8 @@ def test_2d_line_accepts_legend_kw(self): self.darray[:, :, 0].plot.line(x='dim_0', add_legend=True) assert plt.gca().get_legend() # check whether legend title is set - assert plt.gca().get_legend().get_title().get_text() \ - == 'dim_1' + assert (plt.gca().get_legend().get_title().get_text() + == 'dim_1') def test_2d_line_accepts_x_kw(self): self.darray[:, :, 0].plot.line(x='dim_0') @@ -172,12 +174,12 @@ def test_2d_line_accepts_x_kw(self): def test_2d_line_accepts_hue_kw(self): self.darray[:, :, 0].plot.line(hue='dim_0') - assert plt.gca().get_legend().get_title().get_text() \ - == 'dim_0' + assert (plt.gca().get_legend().get_title().get_text() + == 'dim_0') plt.cla() self.darray[:, :, 0].plot.line(hue='dim_1') - assert plt.gca().get_legend().get_title().get_text() \ - == 'dim_1' + assert (plt.gca().get_legend().get_title().get_text() + == 'dim_1') def test_2d_before_squeeze(self): a = DataArray(easy_array((1, 5))) @@ -345,6 +347,7 @@ def test_convenient_facetgrid_4d(self): class TestPlot1D(PlotTestCase): + @pytest.fixture(autouse=True) def setUp(self): d = [0, 1.1, 0, 2] self.darray = DataArray( @@ -357,7 +360,7 @@ def test_xlabel_is_index_name(self): def test_no_label_name_on_x_axis(self): self.darray.plot(y='period') - self.assertEqual('', plt.gca().get_xlabel()) + assert '' == plt.gca().get_xlabel() def test_no_label_name_on_y_axis(self): self.darray.plot() @@ -417,6 +420,7 @@ def test_slice_in_title(self): class TestPlotHistogram(PlotTestCase): + @pytest.fixture(autouse=True) def setUp(self): self.darray = DataArray(easy_array((2, 3, 4))) @@ -452,7 +456,8 @@ def test_plot_nans(self): @requires_matplotlib -class TestDetermineCmapParams(TestCase): +class TestDetermineCmapParams(object): + @pytest.fixture(autouse=True) def setUp(self): self.data = np.linspace(0, 1, num=100) @@ -625,7 +630,8 @@ def test_divergentcontrol(self): @requires_matplotlib -class TestDiscreteColorMap(TestCase): +class TestDiscreteColorMap(object): + @pytest.fixture(autouse=True) def setUp(self): x = np.arange(start=0, stop=10, step=2) y = np.arange(start=9, stop=-7, step=-3) @@ -706,7 +712,7 @@ def test_discrete_colormap_list_levels_and_vmin_or_vmax(self): assert primitive.norm.vmin == min(levels) -class Common2dMixin: +class Common2dMixin(object): """ Common tests for 2d plotting go here. @@ -714,6 +720,7 @@ class Common2dMixin: Should have the same name as the method. """ + @pytest.fixture(autouse=True) def setUp(self): da = DataArray(easy_array((10, 15), start=-1), dims=['y', 'x'], @@ -1145,18 +1152,18 @@ def _color_as_tuple(c): assert artist.cmap.colors[0] == 'k' artist = self.plotmethod(colors=['k', 'b']) - assert _color_as_tuple(artist.cmap.colors[1]) == \ - (0.0, 0.0, 1.0) + assert (_color_as_tuple(artist.cmap.colors[1]) == + (0.0, 0.0, 1.0)) artist = self.darray.plot.contour( levels=[-0.5, 0., 0.5, 1.], colors=['k', 'r', 'w', 'b']) - assert _color_as_tuple(artist.cmap.colors[1]) == \ - (1.0, 0.0, 0.0) - assert _color_as_tuple(artist.cmap.colors[2]) == \ - (1.0, 1.0, 1.0) + assert (_color_as_tuple(artist.cmap.colors[1]) == + (1.0, 0.0, 0.0)) + assert (_color_as_tuple(artist.cmap.colors[2]) == + (1.0, 1.0, 1.0)) # the last color is now under "over" - assert _color_as_tuple(artist.cmap._rgba_over) == \ - (0.0, 0.0, 1.0) + assert (_color_as_tuple(artist.cmap._rgba_over) == + (0.0, 0.0, 1.0)) def test_cmap_and_color_both(self): with pytest.raises(ValueError): @@ -1352,6 +1359,7 @@ def test_origin_overrides_xyincrease(self): class TestFacetGrid(PlotTestCase): + @pytest.fixture(autouse=True) def setUp(self): d = easy_array((10, 15, 3)) self.darray = DataArray( @@ -1581,6 +1589,7 @@ def test_facetgrid_polar(self): @pytest.mark.filterwarnings('ignore:tight_layout cannot') class TestFacetGrid4d(PlotTestCase): + @pytest.fixture(autouse=True) def setUp(self): a = easy_array((10, 15, 3, 2)) darray = DataArray(a, dims=['y', 'x', 'col', 'row']) @@ -1609,6 +1618,7 @@ def test_default_labels(self): @pytest.mark.filterwarnings('ignore:tight_layout cannot') class TestFacetedLinePlots(PlotTestCase): + @pytest.fixture(autouse=True) def setUp(self): self.darray = DataArray(np.random.randn(10, 6, 3, 4), dims=['hue', 'x', 'col', 'row'], @@ -1689,6 +1699,7 @@ def test_wrong_num_of_dimensions(self): class TestDatetimePlot(PlotTestCase): + @pytest.fixture(autouse=True) def setUp(self): ''' Create a DataArray with a time-axis that contains datetime objects. diff --git a/xarray/tests/test_tutorial.py b/xarray/tests/test_tutorial.py index d550a85e8ce..083ec5ee72f 100644 --- a/xarray/tests/test_tutorial.py +++ b/xarray/tests/test_tutorial.py @@ -2,15 +2,17 @@ import os +import pytest + from xarray import DataArray, tutorial from xarray.core.pycompat import suppress -from . import TestCase, assert_identical, network +from . import assert_identical, network @network -class TestLoadDataset(TestCase): - +class TestLoadDataset(object): + @pytest.fixture(autouse=True) def setUp(self): self.testfile = 'tiny' self.testfilepath = os.path.expanduser(os.sep.join( diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 0c0e0f3f744..34f401dd243 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -5,8 +5,8 @@ import numpy as np import pandas as pd import pytest -import xarray as xr +import xarray as xr from xarray.coding.cftimeindex import CFTimeIndex from xarray.core import duck_array_ops, utils from xarray.core.options import set_options @@ -15,12 +15,12 @@ from xarray.testing import assert_identical from . import ( - TestCase, assert_array_equal, has_cftime, has_cftime_or_netCDF4, - requires_dask, requires_cftime) + assert_array_equal, has_cftime, has_cftime_or_netCDF4, requires_cftime, + requires_dask) from .test_coding_times import _all_cftime_date_types -class TestAlias(TestCase): +class TestAlias(object): def test(self): def new_method(): pass @@ -98,7 +98,7 @@ def test_multiindex_from_product_levels_non_unique(): np.testing.assert_array_equal(result.levels[1], [1, 2]) -class TestArrayEquiv(TestCase): +class TestArrayEquiv(object): def test_0d(self): # verify our work around for pd.isnull not working for 0-dimensional # object arrays @@ -108,8 +108,9 @@ def test_0d(self): assert not duck_array_ops.array_equiv(0, np.array(1, dtype=object)) -class TestDictionaries(TestCase): - def setUp(self): +class TestDictionaries(object): + @pytest.fixture(autouse=True) + def setup(self): self.x = {'a': 'A', 'b': 'B'} self.y = {'c': 'C', 'b': 'B'} self.z = {'a': 'Z'} @@ -176,7 +177,7 @@ def test_frozen(self): def test_sorted_keys_dict(self): x = {'a': 1, 'b': 2, 'c': 3} y = utils.SortedKeysDict(x) - self.assertItemsEqual(y, ['a', 'b', 'c']) + assert list(y) == ['a', 'b', 'c'] assert repr(utils.SortedKeysDict()) == \ "SortedKeysDict({})" @@ -191,7 +192,7 @@ def test_chain_map(self): m['x'] = 100 assert m['x'] == 100 assert m.maps[0]['x'] == 100 - self.assertItemsEqual(['x', 'y', 'z'], m) + assert set(m) == {'x', 'y', 'z'} def test_repr_object(): @@ -199,7 +200,7 @@ def test_repr_object(): assert repr(obj) == 'foo' -class Test_is_uniform_and_sorted(TestCase): +class Test_is_uniform_and_sorted(object): def test_sorted_uniform(self): assert utils.is_uniform_spaced(np.arange(5)) @@ -220,7 +221,7 @@ def test_relative_tolerance(self): assert utils.is_uniform_spaced([0, 0.97, 2], rtol=0.1) -class Test_hashable(TestCase): +class Test_hashable(object): def test_hashable(self): for v in [False, 1, (2, ), (3, 4), 'four']: diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 1263ac1df9e..52289a15d72 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1,11 +1,11 @@ from __future__ import absolute_import, division, print_function +import warnings from copy import copy, deepcopy from datetime import datetime, timedelta from distutils.version import LooseVersion from textwrap import dedent -import warnings import numpy as np import pandas as pd @@ -25,11 +25,11 @@ from xarray.tests import requires_bottleneck from . import ( - TestCase, assert_allclose, assert_array_equal, assert_equal, - assert_identical, raises_regex, requires_dask, source_ndarray) + assert_allclose, assert_array_equal, assert_equal, assert_identical, + raises_regex, requires_dask, source_ndarray) -class VariableSubclassTestCases(object): +class VariableSubclassobjects(object): def test_properties(self): data = 0.5 * np.arange(10) v = self.cls(['time'], data, {'foo': 'bar'}) @@ -479,20 +479,20 @@ def test_concat_mixed_dtypes(self): assert_identical(expected, actual) assert actual.dtype == object - def test_copy(self): + @pytest.mark.parametrize('deep', [True, False]) + def test_copy(self, deep): v = self.cls('x', 0.5 * np.arange(10), {'foo': 'bar'}) - for deep in [True, False]: - w = v.copy(deep=deep) - assert type(v) is type(w) - assert_identical(v, w) - assert v.dtype == w.dtype - if self.cls is Variable: - if deep: - assert source_ndarray(v.values) is not \ - source_ndarray(w.values) - else: - assert source_ndarray(v.values) is \ - source_ndarray(w.values) + w = v.copy(deep=deep) + assert type(v) is type(w) + assert_identical(v, w) + assert v.dtype == w.dtype + if self.cls is Variable: + if deep: + assert (source_ndarray(v.values) is not + source_ndarray(w.values)) + else: + assert (source_ndarray(v.values) is + source_ndarray(w.values)) assert_identical(v, copy(v)) def test_copy_index(self): @@ -814,10 +814,11 @@ def test_rolling_window(self): v_loaded[0] = 1.0 -class TestVariable(TestCase, VariableSubclassTestCases): +class TestVariable(VariableSubclassobjects): cls = staticmethod(Variable) - def setUp(self): + @pytest.fixture(autouse=True) + def setup(self): self.d = np.random.random((10, 3)).astype(np.float64) def test_data_and_values(self): @@ -1651,7 +1652,7 @@ def assert_assigned_2d(array, key_x, key_y, values): @requires_dask -class TestVariableWithDask(TestCase, VariableSubclassTestCases): +class TestVariableWithDask(VariableSubclassobjects): cls = staticmethod(lambda *args: Variable(*args).chunk()) @pytest.mark.xfail @@ -1691,7 +1692,7 @@ def test_getitem_with_mask_nd_indexer(self): self.cls(('x', 'y'), [[0, -1], [-1, 2]])) -class TestIndexVariable(TestCase, VariableSubclassTestCases): +class TestIndexVariable(VariableSubclassobjects): cls = staticmethod(IndexVariable) def test_init(self): @@ -1804,7 +1805,7 @@ def test_rolling_window(self): super(TestIndexVariable, self).test_rolling_window() -class TestAsCompatibleData(TestCase): +class TestAsCompatibleData(object): def test_unchanged_types(self): types = (np.asarray, PandasIndexAdapter, LazilyOuterIndexedArray) for t in types: @@ -1945,9 +1946,10 @@ def test_raise_no_warning_for_nan_in_binary_ops(): assert len(record) == 0 -class TestBackendIndexing(TestCase): +class TestBackendIndexing(object): """ Make sure all the array wrappers can be indexed. """ + @pytest.fixture(autouse=True) def setUp(self): self.d = np.random.random((10, 3)).astype(np.float64) From 515324062cf6f182d20c1aad210e8627b0b4013f Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 7 Oct 2018 18:31:16 -0400 Subject: [PATCH 239/282] tests shoudn't need to pass for a PR (#2471) --- .github/PULL_REQUEST_TEMPLATE.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5e9aa06f507..d1c79953a9b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,3 @@ - [ ] Closes #xxxx (remove if there is no corresponding issue, which should only be the case for minor changes) - [ ] Tests added (for all bug fixes or enhancements) - - [ ] Tests passed (for all non-documentation changes) - [ ] Fully documented, including `whats-new.rst` for all changes and `api.rst` for new API (remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later) From 3d65f02de7c0328029dd6c580f42ebeb7381579f Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 7 Oct 2018 18:39:14 -0400 Subject: [PATCH 240/282] isort (#2469) --- asv_bench/benchmarks/dataset_io.py | 2 +- asv_bench/benchmarks/unstacking.py | 1 + properties/test_encode_decode.py | 4 ++-- setup.py | 4 +--- versioneer.py | 10 ++++++---- xarray/backends/netCDF4_.py | 3 +-- xarray/backends/pseudonetcdf_.py | 11 ++++------- xarray/backends/rasterio_.py | 2 +- xarray/coding/cftime_offsets.py | 5 ++--- xarray/coding/cftimeindex.py | 1 + xarray/coding/strings.py | 4 ++-- xarray/coding/times.py | 2 +- xarray/conventions.py | 4 ++-- xarray/core/accessors.py | 2 +- xarray/core/combine.py | 2 +- xarray/core/common.py | 5 ++--- xarray/core/computation.py | 8 ++++---- xarray/core/dask_array_compat.py | 2 +- xarray/core/dask_array_ops.py | 4 ++-- xarray/core/dataset.py | 8 ++++---- xarray/core/formatting.py | 3 +-- xarray/core/missing.py | 7 +++---- xarray/core/nanops.py | 7 +++---- xarray/core/npcompat.py | 1 + xarray/core/resample.py | 2 +- xarray/plot/facetgrid.py | 1 + xarray/plot/utils.py | 1 - xarray/tests/test_cftime_offsets.py | 12 +++++------- xarray/tests/test_cftimeindex.py | 11 +++++------ xarray/tests/test_coding_strings.py | 8 ++++---- xarray/tests/test_coding_times.py | 10 +++++----- xarray/tests/test_computation.py | 2 +- xarray/tests/test_groupby.py | 3 ++- xarray/tests/test_interp.py | 8 ++++---- xarray/tests/test_ufuncs.py | 6 +++--- 35 files changed, 79 insertions(+), 87 deletions(-) diff --git a/asv_bench/benchmarks/dataset_io.py b/asv_bench/benchmarks/dataset_io.py index 54ed9ac9fa2..0b918e58eab 100644 --- a/asv_bench/benchmarks/dataset_io.py +++ b/asv_bench/benchmarks/dataset_io.py @@ -5,7 +5,7 @@ import xarray as xr -from . import randn, randint, requires_dask +from . import randint, randn, requires_dask try: import dask diff --git a/asv_bench/benchmarks/unstacking.py b/asv_bench/benchmarks/unstacking.py index aa304d4eb40..54436b422e9 100644 --- a/asv_bench/benchmarks/unstacking.py +++ b/asv_bench/benchmarks/unstacking.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function import numpy as np + import xarray as xr from . import requires_dask diff --git a/properties/test_encode_decode.py b/properties/test_encode_decode.py index 7b3e75fbf0c..13f63f259cf 100644 --- a/properties/test_encode_decode.py +++ b/properties/test_encode_decode.py @@ -6,9 +6,9 @@ """ from __future__ import absolute_import, division, print_function -from hypothesis import given, settings -import hypothesis.strategies as st import hypothesis.extra.numpy as npst +import hypothesis.strategies as st +from hypothesis import given, settings import xarray as xr diff --git a/setup.py b/setup.py index 68798bdf219..a7519bac6da 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,8 @@ #!/usr/bin/env python import sys -from setuptools import find_packages, setup - import versioneer - +from setuptools import find_packages, setup DISTNAME = 'xarray' LICENSE = 'Apache' diff --git a/versioneer.py b/versioneer.py index 64fea1c8927..dffd66b69a6 100644 --- a/versioneer.py +++ b/versioneer.py @@ -277,10 +277,7 @@ """ from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser + import errno import json import os @@ -288,6 +285,11 @@ import subprocess import sys +try: + import configparser +except ImportError: + import ConfigParser as configparser + class VersioneerConfig: """Container for Versioneer configuration parameters.""" diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 5c6d82fd126..aa19633020b 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -10,8 +10,7 @@ from .. import Variable, coding from ..coding.variables import pop_to from ..core import indexing -from ..core.pycompat import ( - PY3, OrderedDict, basestring, iteritems, suppress) +from ..core.pycompat import PY3, OrderedDict, basestring, iteritems, suppress from ..core.utils import FrozenOrderedDict, close_on_error, is_remote_uri from .common import ( HDF5_LOCK, BackendArray, DataStorePickleMixin, WritableCFDataStore, diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index d946c6fa927..3d846916740 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -1,17 +1,14 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import numpy as np from .. import Variable -from ..core.pycompat import OrderedDict -from ..core.utils import (FrozenOrderedDict, Frozen) from ..core import indexing - -from .common import AbstractDataStore, DataStorePickleMixin, BackendArray +from ..core.pycompat import OrderedDict +from ..core.utils import Frozen, FrozenOrderedDict +from .common import AbstractDataStore, BackendArray, DataStorePickleMixin class PncArrayWrapper(BackendArray): diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 5221cf0e913..9cd5a889abc 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -1,7 +1,7 @@ import os +import warnings from collections import OrderedDict from distutils.version import LooseVersion -import warnings import numpy as np diff --git a/xarray/coding/cftime_offsets.py b/xarray/coding/cftime_offsets.py index 3fbb44f4ed3..83e8c7a7e4b 100644 --- a/xarray/coding/cftime_offsets.py +++ b/xarray/coding/cftime_offsets.py @@ -41,15 +41,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import re - from datetime import timedelta from functools import partial import numpy as np -from .cftimeindex import _parse_iso8601_with_reso, CFTimeIndex -from .times import format_cftime_datetime from ..core.pycompat import basestring +from .cftimeindex import CFTimeIndex, _parse_iso8601_with_reso +from .times import format_cftime_datetime def get_date_type(calendar): diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 75a1fc9bd1a..dea896c199a 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -40,6 +40,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import + import re from datetime import timedelta diff --git a/xarray/coding/strings.py b/xarray/coding/strings.py index 87b17d9175e..3502fd773d7 100644 --- a/xarray/coding/strings.py +++ b/xarray/coding/strings.py @@ -9,8 +9,8 @@ from ..core.pycompat import bytes_type, dask_array_type, unicode_type from ..core.variable import Variable from .variables import ( - VariableCoder, lazy_elemwise_func, pop_to, - safe_setitem, unpack_for_decoding, unpack_for_encoding) + VariableCoder, lazy_elemwise_func, pop_to, safe_setitem, + unpack_for_decoding, unpack_for_encoding) def create_vlen_dtype(element_type): diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 6edbedce54c..dff7e75bdcf 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -9,8 +9,8 @@ import numpy as np import pandas as pd -from ..core.common import contains_cftime_datetimes from ..core import indexing +from ..core.common import contains_cftime_datetimes from ..core.formatting import first_n_items, format_timestamp, last_item from ..core.options import OPTIONS from ..core.pycompat import PY3 diff --git a/xarray/conventions.py b/xarray/conventions.py index 67dcb8d6d4e..f60ee6b2c15 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -6,11 +6,11 @@ import numpy as np import pandas as pd -from .coding import times, strings, variables +from .coding import strings, times, variables from .coding.variables import SerializationWarning from .core import duck_array_ops, indexing from .core.pycompat import ( - OrderedDict, basestring, bytes_type, iteritems, dask_array_type, + OrderedDict, basestring, bytes_type, dask_array_type, iteritems, unicode_type) from .core.variable import IndexVariable, Variable, as_variable diff --git a/xarray/core/accessors.py b/xarray/core/accessors.py index 81af0532d93..72791ed73ec 100644 --- a/xarray/core/accessors.py +++ b/xarray/core/accessors.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from .common import is_np_datetime_like, _contains_datetime_like_objects +from .common import _contains_datetime_like_objects, is_np_datetime_like from .pycompat import dask_array_type diff --git a/xarray/core/combine.py b/xarray/core/combine.py index f0cc025dc7e..6853939c02d 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -8,8 +8,8 @@ from .alignment import align from .merge import merge from .pycompat import OrderedDict, basestring, iteritems -from .variable import concat as concat_vars from .variable import IndexVariable, Variable, as_variable +from .variable import concat as concat_vars def concat(objs, dim=None, data_vars='all', coords='different', diff --git a/xarray/core/common.py b/xarray/core/common.py index 41e4fec2982..c74b1fa080b 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -7,11 +7,10 @@ import numpy as np import pandas as pd -from . import duck_array_ops, dtypes, formatting, ops +from . import dtypes, duck_array_ops, formatting, ops from .arithmetic import SupportsArithmetic from .pycompat import OrderedDict, basestring, dask_array_type, suppress -from .utils import either_dict_or_kwargs, Frozen, SortedKeysDict, ReprObject - +from .utils import Frozen, ReprObject, SortedKeysDict, either_dict_or_kwargs # Used as a sentinel value to indicate a all dimensions ALL_DIMS = ReprObject('') diff --git a/xarray/core/computation.py b/xarray/core/computation.py index bdba72cb48a..7998cc4f72f 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -2,19 +2,19 @@ Functions for applying functions that act on arrays to xarray's labeled data. """ from __future__ import absolute_import, division, print_function -from distutils.version import LooseVersion + import functools import itertools import operator from collections import Counter +from distutils.version import LooseVersion import numpy as np -from . import duck_array_ops -from . import utils +from . import duck_array_ops, utils from .alignment import deep_align from .merge import expand_and_merge_variables -from .pycompat import OrderedDict, dask_array_type, basestring +from .pycompat import OrderedDict, basestring, dask_array_type from .utils import is_dict_like _DEFAULT_FROZEN_SET = frozenset() diff --git a/xarray/core/dask_array_compat.py b/xarray/core/dask_array_compat.py index 2196dba7f86..6b53dcffe6e 100644 --- a/xarray/core/dask_array_compat.py +++ b/xarray/core/dask_array_compat.py @@ -2,9 +2,9 @@ from distutils.version import LooseVersion +import dask.array as da import numpy as np from dask import __version__ as dask_version -import dask.array as da try: from dask.array import isin diff --git a/xarray/core/dask_array_ops.py b/xarray/core/dask_array_ops.py index 423a65aa3c2..25c572edd54 100644 --- a/xarray/core/dask_array_ops.py +++ b/xarray/core/dask_array_ops.py @@ -1,10 +1,10 @@ from __future__ import absolute_import, division, print_function + from distutils.version import LooseVersion import numpy as np -from . import nputils -from . import dtypes +from . import dtypes, nputils try: import dask diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 5e787c1587b..4ade15825c6 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -16,6 +16,7 @@ alignment, computation, duck_array_ops, formatting, groupby, indexing, ops, resample, rolling, utils) from .. import conventions +from ..coding.cftimeindex import _parse_array_of_cftime_strings from .alignment import align from .common import ( ALL_DIMS, DataWithCoords, ImplementsDatasetReduce, @@ -31,12 +32,11 @@ from .pycompat import ( OrderedDict, basestring, dask_array_type, integer_types, iteritems, range) from .utils import ( - Frozen, SortedKeysDict, either_dict_or_kwargs, decode_numpy_dict_values, - ensure_us_time_resolution, hashable, maybe_wrap_array, datetime_to_numeric) + Frozen, SortedKeysDict, datetime_to_numeric, decode_numpy_dict_values, + either_dict_or_kwargs, ensure_us_time_resolution, hashable, + maybe_wrap_array) from .variable import IndexVariable, Variable, as_variable, broadcast_variables -from ..coding.cftimeindex import _parse_array_of_cftime_strings - # list of attributes of pd.DatetimeIndex that are ndarrays of time info _DATETIMEINDEX_COMPONENTS = ['year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond', 'nanosecond', 'date', diff --git a/xarray/core/formatting.py b/xarray/core/formatting.py index 042c8c5324d..5dd3cf06025 100644 --- a/xarray/core/formatting.py +++ b/xarray/core/formatting.py @@ -15,8 +15,7 @@ from .options import OPTIONS from .pycompat import ( - PY2, bytes_type, dask_array_type, unicode_type, zip_longest, -) + PY2, bytes_type, dask_array_type, unicode_type, zip_longest) try: from pandas.errors import OutOfBoundsDatetime diff --git a/xarray/core/missing.py b/xarray/core/missing.py index 0b560c277ae..3f4e0fc3ac9 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -1,20 +1,19 @@ from __future__ import absolute_import, division, print_function +import warnings from collections import Iterable from functools import partial -import warnings - import numpy as np import pandas as pd from . import rolling from .common import _contains_datetime_like_objects from .computation import apply_ufunc +from .duck_array_ops import dask_array_type from .pycompat import iteritems -from .utils import is_scalar, OrderedSet, datetime_to_numeric +from .utils import OrderedSet, datetime_to_numeric, is_scalar from .variable import Variable, broadcast_variables -from .duck_array_ops import dask_array_type class BaseInterpolator(object): diff --git a/xarray/core/nanops.py b/xarray/core/nanops.py index 9549c8e77b9..4d3f03c899e 100644 --- a/xarray/core/nanops.py +++ b/xarray/core/nanops.py @@ -2,11 +2,10 @@ import numpy as np -from . import dtypes +from . import dtypes, nputils +from .duck_array_ops import ( + _dask_or_eager_func, count, fillna, isnull, where_method) from .pycompat import dask_array_type -from . duck_array_ops import (count, isnull, fillna, where_method, - _dask_or_eager_func) -from . import nputils try: import dask.array as dask_array diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index 22dff44acf8..efa68c8bad5 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function from distutils.version import LooseVersion + import numpy as np try: diff --git a/xarray/core/resample.py b/xarray/core/resample.py index 25c149c51af..bd84e04487e 100644 --- a/xarray/core/resample.py +++ b/xarray/core/resample.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function from . import ops -from .groupby import DataArrayGroupBy, DatasetGroupBy, DEFAULT_DIMS +from .groupby import DEFAULT_DIMS, DataArrayGroupBy, DatasetGroupBy from .pycompat import OrderedDict, dask_array_type RESAMPLE_DIM = '__resample_dim__' diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index 79a3993e23b..32a954a3fcd 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -5,6 +5,7 @@ import warnings import numpy as np + from ..core.formatting import format_item from ..core.pycompat import getargspec from .utils import ( diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 455d27c3987..a284c186937 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -8,7 +8,6 @@ from ..core.options import OPTIONS from ..core.pycompat import basestring from ..core.utils import is_scalar -from ..core.options import OPTIONS ROBUST_PERCENTILE = 2.0 diff --git a/xarray/tests/test_cftime_offsets.py b/xarray/tests/test_cftime_offsets.py index 6d7990689ed..7acd764cab3 100644 --- a/xarray/tests/test_cftime_offsets.py +++ b/xarray/tests/test_cftime_offsets.py @@ -1,15 +1,13 @@ -import pytest - from itertools import product import numpy as np +import pytest -from xarray.coding.cftime_offsets import ( - BaseCFTimeOffset, YearBegin, YearEnd, MonthBegin, MonthEnd, - Day, Hour, Minute, Second, _days_in_month, - to_offset, get_date_type, _MONTH_ABBREVIATIONS, to_cftime_datetime, - cftime_range) from xarray import CFTimeIndex +from xarray.coding.cftime_offsets import ( + _MONTH_ABBREVIATIONS, BaseCFTimeOffset, Day, Hour, Minute, MonthBegin, + MonthEnd, Second, YearBegin, YearEnd, _days_in_month, cftime_range, + get_date_type, to_cftime_datetime, to_offset) cftime = pytest.importorskip('cftime') diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index 33bf2cbce0d..d1726ab3313 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -1,16 +1,15 @@ from __future__ import absolute_import -import pytest +from datetime import timedelta import numpy as np import pandas as pd -import xarray as xr +import pytest -from datetime import timedelta +import xarray as xr from xarray.coding.cftimeindex import ( - parse_iso8601, CFTimeIndex, assert_all_valid_date_type, - _parsed_string_to_bounds, _parse_iso8601_with_reso, - _parse_array_of_cftime_strings) + CFTimeIndex, _parse_array_of_cftime_strings, _parse_iso8601_with_reso, + _parsed_string_to_bounds, assert_all_valid_date_type, parse_iso8601) from xarray.tests import assert_array_equal, assert_identical from . import has_cftime, has_cftime_or_netCDF4, requires_cftime diff --git a/xarray/tests/test_coding_strings.py b/xarray/tests/test_coding_strings.py index 53d028e164b..ca138ca8362 100644 --- a/xarray/tests/test_coding_strings.py +++ b/xarray/tests/test_coding_strings.py @@ -5,13 +5,13 @@ import pytest from xarray import Variable -from xarray.core.pycompat import bytes_type, unicode_type, suppress from xarray.coding import strings from xarray.core import indexing +from xarray.core.pycompat import bytes_type, suppress, unicode_type -from . import (IndexerMaker, assert_array_equal, assert_identical, - raises_regex, requires_dask) - +from . import ( + IndexerMaker, assert_array_equal, assert_identical, raises_regex, + requires_dask) with suppress(ImportError): import dask.array as da diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 7d3a4930b44..10a1a956b27 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -1,20 +1,20 @@ from __future__ import absolute_import, division, print_function -from itertools import product import warnings +from itertools import product import numpy as np import pandas as pd import pytest -from xarray import Variable, coding, set_options, DataArray, decode_cf +from xarray import DataArray, Variable, coding, decode_cf, set_options from xarray.coding.times import _import_cftime from xarray.coding.variables import SerializationWarning from xarray.core.common import contains_cftime_datetimes -from . import (assert_array_equal, has_cftime_or_netCDF4, - requires_cftime_or_netCDF4, has_cftime, has_dask) - +from . import ( + assert_array_equal, has_cftime, has_cftime_or_netCDF4, has_dask, + requires_cftime_or_netCDF4) _NON_STANDARD_CALENDARS_SET = {'noleap', '365_day', '360_day', 'julian', 'all_leap', '366_day'} diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index ca8e4e59737..1003c531018 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -15,7 +15,7 @@ join_dict_keys, ordered_set_intersection, ordered_set_union, result_name, unified_dim_sizes) -from . import raises_regex, requires_dask, has_dask +from . import has_dask, raises_regex, requires_dask def assert_identical(a, b): diff --git a/xarray/tests/test_groupby.py b/xarray/tests/test_groupby.py index 6dd14f5d6ad..8ace55be66b 100644 --- a/xarray/tests/test_groupby.py +++ b/xarray/tests/test_groupby.py @@ -5,9 +5,10 @@ import pytest import xarray as xr -from . import assert_identical from xarray.core.groupby import _consolidate_slices +from . import assert_identical + def test_consolidate_slices(): diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py index 0778a1ff128..624879cce1f 100644 --- a/xarray/tests/test_interp.py +++ b/xarray/tests/test_interp.py @@ -5,12 +5,12 @@ import pytest import xarray as xr -from xarray.tests import (assert_allclose, assert_equal, requires_cftime, - requires_scipy) -from . import has_dask, has_scipy -from .test_dataset import create_test_data +from xarray.tests import ( + assert_allclose, assert_equal, requires_cftime, requires_scipy) +from . import has_dask, has_scipy from ..coding.cftimeindex import _parse_array_of_cftime_strings +from .test_dataset import create_test_data try: import scipy diff --git a/xarray/tests/test_ufuncs.py b/xarray/tests/test_ufuncs.py index 195bb36e36e..6941efb1c6e 100644 --- a/xarray/tests/test_ufuncs.py +++ b/xarray/tests/test_ufuncs.py @@ -8,9 +8,9 @@ import xarray as xr import xarray.ufuncs as xu -from . import ( - assert_array_equal, assert_identical as assert_identical_, mock, - raises_regex, requires_np113) +from . import assert_array_equal +from . import assert_identical as assert_identical_ +from . import mock, raises_regex, requires_np113 def assert_identical(a, b): From cf1e6c73d0366124485c1d767b89ac1cc301705b Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Mon, 8 Oct 2018 00:40:07 +0200 Subject: [PATCH 241/282] pep8speaks (#2462) * yml for pep8speaks * Updated yml * Intensionally added a badly styled scripts * experimentally removed yml * .yml file taken from pandas-dev/pandas/.pep8speaks.yml * Undo inteded pep8 violation --- .pep8speaks.yml | 11 +++++++++++ .stickler.yml | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 .pep8speaks.yml delete mode 100644 .stickler.yml diff --git a/.pep8speaks.yml b/.pep8speaks.yml new file mode 100644 index 00000000000..cd610907007 --- /dev/null +++ b/.pep8speaks.yml @@ -0,0 +1,11 @@ +# File : .pep8speaks.yml + +scanner: + diff_only: True # If True, errors caused by only the patch are shown + +pycodestyle: + max-line-length: 79 + ignore: # Errors and warnings to ignore + - E402, # module level import not at top of file + - E731, # do not assign a lambda expression, use a def + - W503 # line break before binary operator diff --git a/.stickler.yml b/.stickler.yml deleted file mode 100644 index 79d8b7fb717..00000000000 --- a/.stickler.yml +++ /dev/null @@ -1,11 +0,0 @@ -linters: - flake8: - max-line-length: 79 - fixer: false - ignore: I002 - # stickler doesn't support 'exclude' for flake8 properly, so we disable it - # below with files.ignore: - # https://github.com/markstory/lint-review/issues/184 -files: - ignore: - - doc/**/*.py From 5f09deb96ac2041e2a2e5affcc8e693bea9a5d73 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 8 Oct 2018 14:23:34 +0900 Subject: [PATCH 242/282] Properly support user-provided norm. (#2443) * Properly support user-provided norm. Fixes #2381 * remove top level mpl import. * More accurate error message. * whats-new fixes. --- doc/whats-new.rst | 13 ++++++---- xarray/plot/plot.py | 12 ++++++---- xarray/plot/utils.py | 33 +++++++++++++++++++++++--- xarray/tests/test_plot.py | 50 ++++++++++++++++++++++++++++++++------- 4 files changed, 87 insertions(+), 21 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e9c223ff801..85e9f2313d6 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -40,15 +40,15 @@ Breaking changes Documentation ~~~~~~~~~~~~~ + Enhancements ~~~~~~~~~~~~ - Added support for Python 3.7. (:issue:`2271`). By `Joe Hamman `_. - - Added :py:meth:`~xarray.CFTimeIndex.shift` for shifting the values of a - CFTimeIndex by a specified frequency. (:issue:`2244`). By `Spencer Clark - `_. + CFTimeIndex by a specified frequency. (:issue:`2244`). + By `Spencer Clark `_. - Added support for using ``cftime.datetime`` coordinates with :py:meth:`~xarray.DataArray.differentiate`, :py:meth:`~xarray.Dataset.differentiate`, @@ -60,11 +60,14 @@ Bug fixes ~~~~~~~~~ - Addition and subtraction operators used with a CFTimeIndex now preserve the - index's type. (:issue:`2244`). By `Spencer Clark `_. + index's type. (:issue:`2244`). + By `Spencer Clark `_. - ``xarray.DataArray.roll`` correctly handles multidimensional arrays. (:issue:`2445`) By `Keisuke Fujii `_. - +- ``xarray.plot()`` now properly accepts a ``norm`` argument and does not override + the norm's ``vmin`` and ``vmax``. (:issue:`2381`) + By `Deepak Cherian `_. - ``xarray.DataArray.std()`` now correctly accepts ``ddof`` keyword argument. (:issue:`2240`) By `Keisuke Fujii `_. diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 3f9f1090c70..b44ae7b3856 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -562,6 +562,9 @@ def _plot2d(plotfunc): Adds colorbar to axis add_labels : Boolean, optional Use xarray metadata to label axes + norm : ``matplotlib.colors.Normalize`` instance, optional + If the ``norm`` has vmin or vmax specified, the corresponding kwarg + must be None. vmin, vmax : floats, optional Values to anchor the colormap, otherwise they are inferred from the data and other keyword arguments. When a diverging dataset is inferred, @@ -630,7 +633,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, levels=None, infer_intervals=None, colors=None, subplot_kws=None, cbar_ax=None, cbar_kwargs=None, xscale=None, yscale=None, xticks=None, yticks=None, - xlim=None, ylim=None, **kwargs): + xlim=None, ylim=None, norm=None, **kwargs): # All 2d plots in xarray share this function signature. # Method signature below should be consistent. @@ -727,6 +730,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, 'extend': extend, 'levels': levels, 'filled': plotfunc.__name__ != 'contour', + 'norm': norm, } cmap_params = _determine_cmap_params(**cmap_kwargs) @@ -746,9 +750,6 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, if 'pcolormesh' == plotfunc.__name__: kwargs['infer_intervals'] = infer_intervals - # This allows the user to pass in a custom norm coming via kwargs - kwargs.setdefault('norm', cmap_params['norm']) - if 'imshow' == plotfunc.__name__ and isinstance(aspect, basestring): # forbid usage of mpl strings raise ValueError("plt.imshow's `aspect` kwarg is not available " @@ -758,6 +759,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, primitive = plotfunc(xval, yval, zval, ax=ax, cmap=cmap_params['cmap'], vmin=cmap_params['vmin'], vmax=cmap_params['vmax'], + norm=cmap_params['norm'], **kwargs) # Label the plot with metadata @@ -809,7 +811,7 @@ def plotmethod(_PlotMethods_obj, x=None, y=None, figsize=None, size=None, levels=None, infer_intervals=None, subplot_kws=None, cbar_ax=None, cbar_kwargs=None, xscale=None, yscale=None, xticks=None, yticks=None, - xlim=None, ylim=None, **kwargs): + xlim=None, ylim=None, norm=None, **kwargs): """ The method should have the same signature as the function. diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index a284c186937..be38a6d7a4c 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -172,6 +172,10 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, # vlim might be computed below vlim = None + # save state; needed later + vmin_was_none = vmin is None + vmax_was_none = vmax is None + if vmin is None: if robust: vmin = np.percentile(calc_data, ROBUST_PERCENTILE) @@ -204,6 +208,28 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, vmin += center vmax += center + # now check norm and harmonize with vmin, vmax + if norm is not None: + if norm.vmin is None: + norm.vmin = vmin + else: + if not vmin_was_none and vmin != norm.vmin: + raise ValueError('Cannot supply vmin and a norm' + + ' with a different vmin.') + vmin = norm.vmin + + if norm.vmax is None: + norm.vmax = vmax + else: + if not vmax_was_none and vmax != norm.vmax: + raise ValueError('Cannot supply vmax and a norm' + + ' with a different vmax.') + vmax = norm.vmax + + # if BoundaryNorm, then set levels + if isinstance(norm, mpl.colors.BoundaryNorm): + levels = norm.boundaries + # Choose default colormaps if not provided if cmap is None: if divergent: @@ -212,7 +238,7 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, cmap = OPTIONS['cmap_sequential'] # Handle discrete levels - if levels is not None: + if levels is not None and norm is None: if is_scalar(levels): if user_minmax: levels = np.linspace(vmin, vmax, levels) @@ -227,8 +253,9 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, if extend is None: extend = _determine_extend(calc_data, vmin, vmax) - if levels is not None: - cmap, norm = _build_discrete_cmap(cmap, levels, extend, filled) + if levels is not None or isinstance(norm, mpl.colors.BoundaryNorm): + cmap, newnorm = _build_discrete_cmap(cmap, levels, extend, filled) + norm = newnorm if norm is None else norm return dict(vmin=vmin, vmax=vmax, cmap=cmap, extend=extend, levels=levels, norm=norm) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 01303202c93..53f6077ee4f 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -628,6 +628,26 @@ def test_divergentcontrol(self): assert cmap_params['vmax'] == 0.6 assert cmap_params['cmap'] == "viridis" + def test_norm_sets_vmin_vmax(self): + vmin = self.data.min() + vmax = self.data.max() + + for norm, extend in zip([mpl.colors.LogNorm(), + mpl.colors.LogNorm(vmin + 1, vmax - 1), + mpl.colors.LogNorm(None, vmax - 1), + mpl.colors.LogNorm(vmin + 1, None)], + ['neither', 'both', 'max', 'min']): + + test_min = vmin if norm.vmin is None else norm.vmin + test_max = vmax if norm.vmax is None else norm.vmax + + cmap_params = _determine_cmap_params(self.data, norm=norm) + + assert cmap_params['vmin'] == test_min + assert cmap_params['vmax'] == test_max + assert cmap_params['extend'] == extend + assert cmap_params['norm'] == norm + @requires_matplotlib class TestDiscreteColorMap(object): @@ -665,10 +685,10 @@ def test_build_discrete_cmap(self): @pytest.mark.slow def test_discrete_colormap_list_of_levels(self): - for extend, levels in [('max', [-1, 2, 4, 8, 10]), ('both', - [2, 5, 10, 11]), - ('neither', [0, 5, 10, 15]), ('min', - [2, 5, 10, 15])]: + for extend, levels in [('max', [-1, 2, 4, 8, 10]), + ('both', [2, 5, 10, 11]), + ('neither', [0, 5, 10, 15]), + ('min', [2, 5, 10, 15])]: for kind in ['imshow', 'pcolormesh', 'contourf', 'contour']: primitive = getattr(self.darray.plot, kind)(levels=levels) assert_array_equal(levels, primitive.norm.boundaries) @@ -682,10 +702,10 @@ def test_discrete_colormap_list_of_levels(self): @pytest.mark.slow def test_discrete_colormap_int_levels(self): - for extend, levels, vmin, vmax in [('neither', 7, None, - None), ('neither', 7, None, 20), - ('both', 7, 4, 8), ('min', 10, 4, - 15)]: + for extend, levels, vmin, vmax in [('neither', 7, None, None), + ('neither', 7, None, 20), + ('both', 7, 4, 8), + ('min', 10, 4, 15)]: for kind in ['imshow', 'pcolormesh', 'contourf', 'contour']: primitive = getattr(self.darray.plot, kind)( levels=levels, vmin=vmin, vmax=vmax) @@ -711,6 +731,11 @@ def test_discrete_colormap_list_levels_and_vmin_or_vmax(self): assert primitive.norm.vmax == max(levels) assert primitive.norm.vmin == min(levels) + def test_discrete_colormap_provided_boundary_norm(self): + norm = mpl.colors.BoundaryNorm([0, 5, 10, 15], 4) + primitive = self.darray.plot.contourf(norm=norm) + np.testing.assert_allclose(primitive.levels, norm.boundaries) + class Common2dMixin(object): """ @@ -1085,6 +1110,15 @@ def test_cmap_and_color_both(self): with pytest.raises(ValueError): self.plotmethod(colors='k', cmap='RdBu') + def test_colormap_error_norm_and_vmin_vmax(self): + norm = mpl.colors.LogNorm(0.1, 1e1) + + with pytest.raises(ValueError): + self.darray.plot(norm=norm, vmin=2) + + with pytest.raises(ValueError): + self.darray.plot(norm=norm, vmax=2) + @pytest.mark.slow class TestContourf(Common2dMixin, PlotTestCase): From 5b4d160d9a714c2cc83ff5788e2d73af92129713 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 8 Oct 2018 11:17:01 -0700 Subject: [PATCH 243/282] Fix indexing error for data loaded with open_rasterio (#2456) xref GH2454 --- doc/whats-new.rst | 6 +++++- xarray/backends/rasterio_.py | 2 +- xarray/tests/test_backends.py | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 85e9f2313d6..7cdb1685f5f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -76,6 +76,10 @@ Bug fixes By `Deepak Cherian `_. +- Fix a bug that caused some indexing operations on arrays opened with + ``open_rasterio`` to error (:issue:`2454`). + By `Stephan Hoyer `_. + .. _whats-new.0.10.9: v0.10.9 (21 September 2018) @@ -86,7 +90,7 @@ This minor release contains a number of backwards compatible enhancements. Announcements of note: - Xarray is now a NumFOCUS fiscally sponsored project! Read - `the anouncment `_ + `the anouncement `_ for more details. - We have a new :doc:`roadmap` that outlines our future development plans. diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 9cd5a889abc..44cca9aaaf8 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -95,7 +95,7 @@ def _get_indexer(self, key): if isinstance(key[1], np.ndarray) and isinstance(key[2], np.ndarray): # do outer-style indexing - np_inds[1:] = np.ix_(*np_inds[1:]) + np_inds[-2:] = np.ix_(*np_inds[-2:]) return band_key, tuple(window), tuple(squeeze_axis), tuple(np_inds) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a2e1cb4c0fa..0d97ed70fa3 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2925,6 +2925,10 @@ def test_indexing(self): assert_allclose(expected.isel(**ind), actual.isel(**ind)) assert not actual.variable._in_memory + ind = {'band': 0, 'x': np.array([0, 0]), 'y': np.array([1, 1, 1])} + assert_allclose(expected.isel(**ind), actual.isel(**ind)) + assert not actual.variable._in_memory + # minus-stepped slice ind = {'band': np.array([2, 1, 0]), 'x': slice(-1, None, -1), 'y': 0} From 289b377129b18e7dc6da8336e958a85be868acbe Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 8 Oct 2018 21:13:41 -0700 Subject: [PATCH 244/282] xarray.backends refactor (#2261) * WIP: xarray.backends.file_manager for managing file objects. This is intended to replace both PickleByReconstructionWrapper and DataStorePickleMixin with something more compartmentalized. xref GH2121 * Switch rasterio to use FileManager * lint fixes * WIP: rewrite FileManager to always use an LRUCache * Test coverage * Don't use move_to_end * minor clarification * Switch FileManager.acquire() to a method * Python 2 compat * Update xarray.set_options() to add file_cache_maxsize and validation * Add assert for FILE_CACHE.maxsize * More docstring for FileManager * Add accidentally omited tests for LRUCache * Adapt scipy backend to use FileManager * Stickler fix * Fix failure on Python 2.7 * Finish adjusting backends to use FileManager * Fix bad import * WIP on distributed * More WIP * Fix distributed write tests * Fixes * Minor fixup * whats new * More refactoring: remove state from backends entirely * Cleanup * Fix failing in-memory datastore tests * Fix inaccessible datastore * fix autoclose warnings * Fix PyNIO failures * No longer disable HDF5 file locking We longer need to explicitly HDF5_USE_FILE_LOCKING='FALSE' because we properly close open files. * whats new and default file cache size * Whats new tweak * Refactor default lock logic to backend classes * Rename get_resource_lock -> get_write_lock * Don't acquire unnecessary locks in __getitem__ * Fix bad merge * Fix import * Remove unreachable code --- asv_bench/asv.conf.json | 1 + asv_bench/benchmarks/dataset_io.py | 41 ++++ doc/api.rst | 3 + doc/whats-new.rst | 19 +- xarray/backends/__init__.py | 4 + xarray/backends/api.py | 250 ++++++++++----------- xarray/backends/common.py | 215 +----------------- xarray/backends/file_manager.py | 206 +++++++++++++++++ xarray/backends/h5netcdf_.py | 169 +++++++------- xarray/backends/locks.py | 191 ++++++++++++++++ xarray/backends/lru_cache.py | 91 ++++++++ xarray/backends/memory.py | 3 +- xarray/backends/netCDF4_.py | 231 ++++++++++--------- xarray/backends/pseudonetcdf_.py | 79 ++++--- xarray/backends/pynio_.py | 53 ++--- xarray/backends/rasterio_.py | 94 ++++---- xarray/backends/scipy_.py | 129 +++++------ xarray/backends/zarr.py | 21 +- xarray/core/dataset.py | 25 +-- xarray/core/options.py | 71 ++++-- xarray/core/pycompat.py | 9 +- xarray/tests/test_backends.py | 209 +++++------------ xarray/tests/test_backends_file_manager.py | 114 ++++++++++ xarray/tests/test_backends_locks.py | 13 ++ xarray/tests/test_backends_lru_cache.py | 91 ++++++++ xarray/tests/test_dataset.py | 4 +- xarray/tests/test_distributed.py | 110 +++++---- xarray/tests/test_options.py | 33 +++ 28 files changed, 1496 insertions(+), 983 deletions(-) create mode 100644 xarray/backends/file_manager.py create mode 100644 xarray/backends/locks.py create mode 100644 xarray/backends/lru_cache.py create mode 100644 xarray/tests/test_backends_file_manager.py create mode 100644 xarray/tests/test_backends_locks.py create mode 100644 xarray/tests/test_backends_lru_cache.py diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json index b5953436387..e3933b400e6 100644 --- a/asv_bench/asv.conf.json +++ b/asv_bench/asv.conf.json @@ -64,6 +64,7 @@ "scipy": [""], "bottleneck": ["", null], "dask": [""], + "distributed": [""], }, diff --git a/asv_bench/benchmarks/dataset_io.py b/asv_bench/benchmarks/dataset_io.py index 0b918e58eab..da18d541a16 100644 --- a/asv_bench/benchmarks/dataset_io.py +++ b/asv_bench/benchmarks/dataset_io.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, division, print_function +import os + import numpy as np import pandas as pd @@ -14,6 +16,9 @@ pass +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' + + class IOSingleNetCDF(object): """ A few examples that benchmark reading/writing a single netCDF file with @@ -405,3 +410,39 @@ def time_open_dataset_scipy_with_time_chunks(self): with dask.set_options(get=dask.multiprocessing.get): xr.open_mfdataset(self.filenames_list, engine='scipy', chunks=self.time_chunks) + + +def create_delayed_write(): + import dask.array as da + vals = da.random.random(300, chunks=(1,)) + ds = xr.Dataset({'vals': (['a'], vals)}) + return ds.to_netcdf('file.nc', engine='netcdf4', compute=False) + + +class IOWriteNetCDFDask(object): + timeout = 60 + repeat = 1 + number = 5 + + def setup(self): + requires_dask() + self.write = create_delayed_write() + + def time_write(self): + self.write.compute() + + +class IOWriteNetCDFDaskDistributed(object): + def setup(self): + try: + import distributed + except ImportError: + raise NotImplementedError + self.client = distributed.Client() + self.write = create_delayed_write() + + def cleanup(self): + self.client.shutdown() + + def time_write(self): + self.write.compute() diff --git a/doc/api.rst b/doc/api.rst index d204fab3539..662ef567710 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -624,3 +624,6 @@ arguments for the ``from_store`` and ``dump_to_store`` Dataset methods: backends.H5NetCDFStore backends.PydapDataStore backends.ScipyDataStore + backends.FileManager + backends.CachingFileManager + backends.DummyFileManager diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 7cdb1685f5f..d0fec7b0778 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,14 +33,27 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ +- Xarray's storage backends now automatically open and close files when + necessary, rather than requiring opening a file with ``autoclose=True``. A + global least-recently-used cache is used to store open files; the default + limit of 128 open files should suffice in most cases, but can be adjusted if + necessary with + ``xarray.set_options(file_cache_maxsize=...)``. The ``autoclose`` argument + to ``open_dataset`` and related functions has been deprecated and is now a + no-op. + + This change, along with an internal refactor of xarray's storage backends, + should significantly improve performance when reading and writing + netCDF files with Dask, especially when working with many files or using + Dask Distributed. By `Stephan Hoyer `_ + +Documentation +~~~~~~~~~~~~~ - Reduction of :py:meth:`DataArray.groupby` and :py:meth:`DataArray.resample` without dimension argument will change in the next release. Now we warn a FutureWarning. By `Keisuke Fujii `_. -Documentation -~~~~~~~~~~~~~ - Enhancements ~~~~~~~~~~~~ diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index 47a2011a3af..a2f0d79a6d1 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -4,6 +4,7 @@ formats. They should not be used directly, but rather through Dataset objects. """ from .common import AbstractDataStore +from .file_manager import FileManager, CachingFileManager, DummyFileManager from .memory import InMemoryDataStore from .netCDF4_ import NetCDF4DataStore from .pydap_ import PydapDataStore @@ -15,6 +16,9 @@ __all__ = [ 'AbstractDataStore', + 'FileManager', + 'CachingFileManager', + 'DummyFileManager', 'InMemoryDataStore', 'NetCDF4DataStore', 'PydapDataStore', diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 2bf13011bd1..65112527045 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -4,6 +4,7 @@ from glob import glob from io import BytesIO from numbers import Number +import warnings import numpy as np @@ -12,8 +13,9 @@ from ..core.combine import auto_combine from ..core.pycompat import basestring, path_type from ..core.utils import close_on_error, is_remote_uri -from .common import ( - HDF5_LOCK, ArrayWriter, CombinedLock, _get_scheduler, _get_scheduler_lock) +from .common import ArrayWriter +from .locks import _get_scheduler + DATAARRAY_NAME = '__xarray_dataarray_name__' DATAARRAY_VARIABLE = '__xarray_dataarray_variable__' @@ -52,27 +54,6 @@ def _normalize_path(path): return os.path.abspath(os.path.expanduser(path)) -def _default_lock(filename, engine): - if filename.endswith('.gz'): - lock = False - else: - if engine is None: - engine = _get_default_engine(filename, allow_remote=True) - - if engine == 'netcdf4': - if is_remote_uri(filename): - lock = False - else: - # TODO: identify netcdf3 files and don't use the global lock - # for them - lock = HDF5_LOCK - elif engine in {'h5netcdf', 'pynio'}: - lock = HDF5_LOCK - else: - lock = False - return lock - - def _validate_dataset_names(dataset): """DataArray.name and Dataset keys must be a string or None""" def check_name(name): @@ -130,29 +111,14 @@ def _protect_dataset_variables_inplace(dataset, cache): variable.data = data -def _get_lock(engine, scheduler, format, path_or_file): - """ Get the lock(s) that apply to a particular scheduler/engine/format""" - - locks = [] - if format in ['NETCDF4', None] and engine in ['h5netcdf', 'netcdf4']: - locks.append(HDF5_LOCK) - locks.append(_get_scheduler_lock(scheduler, path_or_file)) - - # When we have more than one lock, use the CombinedLock wrapper class - lock = CombinedLock(locks) if len(locks) > 1 else locks[0] - - return lock - - def _finalize_store(write, store): """ Finalize this store by explicitly syncing and closing""" del write # ensure writing is done first - store.sync() store.close() def open_dataset(filename_or_obj, group=None, decode_cf=True, - mask_and_scale=None, decode_times=True, autoclose=False, + mask_and_scale=None, decode_times=True, autoclose=None, concat_characters=True, decode_coords=True, engine=None, chunks=None, lock=None, cache=None, drop_variables=None, backend_kwargs=None): @@ -204,12 +170,11 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, If chunks is provided, it used to load the new dataset into dask arrays. ``chunks={}`` loads the dataset with dask using a single chunk for all arrays. - lock : False, True or threading.Lock, optional - If chunks is provided, this argument is passed on to - :py:func:`dask.array.from_array`. By default, a global lock is - used when reading data from netCDF files with the netcdf4 and h5netcdf - engines to avoid issues with concurrent access when using dask's - multithreaded backend. + lock : False or duck threading.Lock, optional + Resource lock to use when reading data from disk. Only relevant when + using dask or another form of parallelism. By default, appropriate + locks are chosen to safely read and write files with the currently + active dask scheduler. cache : bool, optional If True, cache data loaded from the underlying datastore in memory as NumPy arrays when accessed to avoid reading from the underlying data- @@ -235,6 +200,14 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, -------- open_mfdataset """ + if autoclose is not None: + warnings.warn( + 'The autoclose argument is no longer used by ' + 'xarray.open_dataset() and is now ignored; it will be removed in ' + 'xarray v0.12. If necessary, you can control the maximum number ' + 'of simultaneous open files with ' + 'xarray.set_options(file_cache_maxsize=...).', + FutureWarning, stacklevel=2) if mask_and_scale is None: mask_and_scale = not engine == 'pseudonetcdf' @@ -272,18 +245,11 @@ def maybe_decode_store(store, lock=False): mask_and_scale, decode_times, concat_characters, decode_coords, engine, chunks, drop_variables) name_prefix = 'open_dataset-%s' % token - ds2 = ds.chunk(chunks, name_prefix=name_prefix, token=token, - lock=lock) + ds2 = ds.chunk(chunks, name_prefix=name_prefix, token=token) ds2._file_obj = ds._file_obj else: ds2 = ds - # protect so that dataset store isn't necessarily closed, e.g., - # streams like BytesIO can't be reopened - # datastore backend is responsible for determining this capability - if store._autoclose: - store.close() - return ds2 if isinstance(filename_or_obj, path_type): @@ -314,36 +280,28 @@ def maybe_decode_store(store, lock=False): engine = _get_default_engine(filename_or_obj, allow_remote=True) if engine == 'netcdf4': - store = backends.NetCDF4DataStore.open(filename_or_obj, - group=group, - autoclose=autoclose, - **backend_kwargs) + store = backends.NetCDF4DataStore.open( + filename_or_obj, group=group, lock=lock, **backend_kwargs) elif engine == 'scipy': - store = backends.ScipyDataStore(filename_or_obj, - autoclose=autoclose, - **backend_kwargs) + store = backends.ScipyDataStore(filename_or_obj, **backend_kwargs) elif engine == 'pydap': - store = backends.PydapDataStore.open(filename_or_obj, - **backend_kwargs) + store = backends.PydapDataStore.open( + filename_or_obj, **backend_kwargs) elif engine == 'h5netcdf': - store = backends.H5NetCDFStore(filename_or_obj, group=group, - autoclose=autoclose, - **backend_kwargs) + store = backends.H5NetCDFStore( + filename_or_obj, group=group, lock=lock, **backend_kwargs) elif engine == 'pynio': - store = backends.NioDataStore(filename_or_obj, - autoclose=autoclose, - **backend_kwargs) + store = backends.NioDataStore( + filename_or_obj, lock=lock, **backend_kwargs) elif engine == 'pseudonetcdf': store = backends.PseudoNetCDFDataStore.open( - filename_or_obj, autoclose=autoclose, **backend_kwargs) + filename_or_obj, lock=lock, **backend_kwargs) else: raise ValueError('unrecognized engine for open_dataset: %r' % engine) - if lock is None: - lock = _default_lock(filename_or_obj, engine) with close_on_error(store): - return maybe_decode_store(store, lock) + return maybe_decode_store(store) else: if engine is not None and engine != 'scipy': raise ValueError('can only read file-like objects with ' @@ -355,7 +313,7 @@ def maybe_decode_store(store, lock=False): def open_dataarray(filename_or_obj, group=None, decode_cf=True, - mask_and_scale=None, decode_times=True, autoclose=False, + mask_and_scale=None, decode_times=True, autoclose=None, concat_characters=True, decode_coords=True, engine=None, chunks=None, lock=None, cache=None, drop_variables=None, backend_kwargs=None): @@ -390,10 +348,6 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, decode_times : bool, optional If True, decode times encoded in the standard NetCDF datetime format into datetime objects. Otherwise, leave them encoded as numbers. - autoclose : bool, optional - If True, automatically close files to avoid OS Error of too many files - being open. However, this option doesn't work with streams, e.g., - BytesIO. concat_characters : bool, optional If True, concatenate along the last dimension of character arrays to form string arrays. Dimensions will only be concatenated over (and @@ -409,12 +363,11 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, chunks : int or dict, optional If chunks is provided, it used to load the new dataset into dask arrays. - lock : False, True or threading.Lock, optional - If chunks is provided, this argument is passed on to - :py:func:`dask.array.from_array`. By default, a global lock is - used when reading data from netCDF files with the netcdf4 and h5netcdf - engines to avoid issues with concurrent access when using dask's - multithreaded backend. + lock : False or duck threading.Lock, optional + Resource lock to use when reading data from disk. Only relevant when + using dask or another form of parallelism. By default, appropriate + locks are chosen to safely read and write files with the currently + active dask scheduler. cache : bool, optional If True, cache data loaded from the underlying datastore in memory as NumPy arrays when accessed to avoid reading from the underlying data- @@ -490,7 +443,7 @@ def close(self): def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, compat='no_conflicts', preprocess=None, engine=None, lock=None, data_vars='all', coords='different', - autoclose=False, parallel=False, **kwargs): + autoclose=None, parallel=False, **kwargs): """Open multiple files as a single dataset. Requires dask to be installed. See documentation for details on dask [1]. @@ -537,15 +490,11 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, Engine to use when reading files. If not provided, the default engine is chosen based on available dependencies, with a preference for 'netcdf4'. - autoclose : bool, optional - If True, automatically close files to avoid OS Error of too many files - being open. However, this option doesn't work with streams, e.g., - BytesIO. - lock : False, True or threading.Lock, optional - This argument is passed on to :py:func:`dask.array.from_array`. By - default, a per-variable lock is used when reading data from netCDF - files with the netcdf4 and h5netcdf engines to avoid issues with - concurrent access when using dask's multithreaded backend. + lock : False or duck threading.Lock, optional + Resource lock to use when reading data from disk. Only relevant when + using dask or another form of parallelism. By default, appropriate + locks are chosen to safely read and write files with the currently + active dask scheduler. data_vars : {'minimal', 'different', 'all' or list of str}, optional These data variables will be concatenated together: * 'minimal': Only data variables in which the dimension already @@ -604,9 +553,6 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, if not paths: raise IOError('no files to open') - if lock is None: - lock = _default_lock(paths[0], engine) - open_kwargs = dict(engine=engine, chunks=chunks or {}, lock=lock, autoclose=autoclose, **kwargs) @@ -656,19 +602,21 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, - engine=None, writer=None, encoding=None, unlimited_dims=None, - compute=True): + engine=None, encoding=None, unlimited_dims=None, compute=True, + multifile=False): """This function creates an appropriate datastore for writing a dataset to disk as a netCDF file See `Dataset.to_netcdf` for full API docs. - The ``writer`` argument is only for the private use of save_mfdataset. + The ``multifile`` argument is only for the private use of save_mfdataset. """ if isinstance(path_or_file, path_type): path_or_file = str(path_or_file) + if encoding is None: encoding = {} + if path_or_file is None: if engine is None: engine = 'scipy' @@ -676,6 +624,10 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, raise ValueError('invalid engine for creating bytes with ' 'to_netcdf: %r. Only the default engine ' "or engine='scipy' is supported" % engine) + if not compute: + raise NotImplementedError( + 'to_netcdf() with compute=False is not yet implemented when ' + 'returning bytes') elif isinstance(path_or_file, basestring): if engine is None: engine = _get_default_engine(path_or_file) @@ -695,45 +647,78 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, if format is not None: format = format.upper() - # if a writer is provided, store asynchronously - sync = writer is None - # handle scheduler specific logic scheduler = _get_scheduler() have_chunks = any(v.chunks for v in dataset.variables.values()) - if (have_chunks and scheduler in ['distributed', 'multiprocessing'] and - engine != 'netcdf4'): + + autoclose = have_chunks and scheduler in ['distributed', 'multiprocessing'] + if autoclose and engine == 'scipy': raise NotImplementedError("Writing netCDF files with the %s backend " "is not currently supported with dask's %s " "scheduler" % (engine, scheduler)) - lock = _get_lock(engine, scheduler, format, path_or_file) - autoclose = (have_chunks and - scheduler in ['distributed', 'multiprocessing']) target = path_or_file if path_or_file is not None else BytesIO() - store = store_open(target, mode, format, group, writer, - autoclose=autoclose, lock=lock) + kwargs = dict(autoclose=True) if autoclose else {} + store = store_open(target, mode, format, group, **kwargs) if unlimited_dims is None: unlimited_dims = dataset.encoding.get('unlimited_dims', None) if isinstance(unlimited_dims, basestring): unlimited_dims = [unlimited_dims] + writer = ArrayWriter() + + # TODO: figure out how to refactor this logic (here and in save_mfdataset) + # to avoid this mess of conditionals try: - dataset.dump_to_store(store, sync=sync, encoding=encoding, - unlimited_dims=unlimited_dims, compute=compute) + # TODO: allow this work (setting up the file for writing array data) + # to be parallelized with dask + dump_to_store(dataset, store, writer, encoding=encoding, + unlimited_dims=unlimited_dims) + if autoclose: + store.close() + + if multifile: + return writer, store + + writes = writer.sync(compute=compute) + if path_or_file is None: + store.sync() return target.getvalue() finally: - if sync and isinstance(path_or_file, basestring): + if not multifile and compute: store.close() if not compute: import dask - return dask.delayed(_finalize_store)(store.delayed_store, store) + return dask.delayed(_finalize_store)(writes, store) + + +def dump_to_store(dataset, store, writer=None, encoder=None, + encoding=None, unlimited_dims=None): + """Store dataset contents to a backends.*DataStore object.""" + if writer is None: + writer = ArrayWriter() + + if encoding is None: + encoding = {} + + variables, attrs = conventions.encode_dataset_coordinates(dataset) + + check_encoding = set() + for k, enc in encoding.items(): + # no need to shallow copy the variable again; that already happened + # in encode_dataset_coordinates + variables[k].encoding = enc + check_encoding.add(k) + + if encoder: + variables, attrs = encoder(variables, attrs) + + store.store(variables, attrs, check_encoding, writer, + unlimited_dims=unlimited_dims) - if not sync: - return store def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, engine=None, compute=True): @@ -816,22 +801,22 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, 'datasets, paths and groups arguments to ' 'save_mfdataset') - writer = ArrayWriter() if compute else None - stores = [to_netcdf(ds, path, mode, format, group, engine, writer, - compute=compute) - for ds, path, group in zip(datasets, paths, groups)] - - if not compute: - import dask - return dask.delayed(stores) + writers, stores = zip(*[ + to_netcdf(ds, path, mode, format, group, engine, compute=compute, + multifile=True) + for ds, path, group in zip(datasets, paths, groups)]) try: - delayed = writer.sync(compute=compute) - for store in stores: - store.sync() + writes = [w.sync(compute=compute) for w in writers] finally: - for store in stores: - store.close() + if compute: + for store in stores: + store.close() + + if not compute: + import dask + return dask.delayed([dask.delayed(_finalize_store)(w, s) + for w, s in zip(writes, stores)]) def to_zarr(dataset, store=None, mode='w-', synchronizer=None, group=None, @@ -852,13 +837,14 @@ def to_zarr(dataset, store=None, mode='w-', synchronizer=None, group=None, store = backends.ZarrStore.open_group(store=store, mode=mode, synchronizer=synchronizer, - group=group, writer=None) + group=group) - # I think zarr stores should always be sync'd immediately + writer = ArrayWriter() # TODO: figure out how to properly handle unlimited_dims - dataset.dump_to_store(store, sync=True, encoding=encoding, compute=compute) + dump_to_store(dataset, store, writer, encoding=encoding) + writes = writer.sync(compute=compute) if not compute: import dask - return dask.delayed(_finalize_store)(store.delayed_store, store) + return dask.delayed(_finalize_store)(writes, store) return store diff --git a/xarray/backends/common.py b/xarray/backends/common.py index 99f7698ee92..405d989f4af 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -1,14 +1,10 @@ from __future__ import absolute_import, division, print_function -import contextlib import logging -import multiprocessing -import threading import time import traceback import warnings from collections import Mapping, OrderedDict -from functools import partial import numpy as np @@ -17,13 +13,6 @@ from ..core.pycompat import dask_array_type, iteritems from ..core.utils import FrozenOrderedDict, NdimSizeLenMixin -# Import default lock -try: - from dask.utils import SerializableLock - HDF5_LOCK = SerializableLock() -except ImportError: - HDF5_LOCK = threading.Lock() - # Create a logger object, but don't add any handlers. Leave that to user code. logger = logging.getLogger(__name__) @@ -31,62 +20,6 @@ NONE_VAR_NAME = '__values__' -def _get_scheduler(get=None, collection=None): - """ Determine the dask scheduler that is being used. - - None is returned if not dask scheduler is active. - - See also - -------- - dask.base.get_scheduler - """ - try: - # dask 0.18.1 and later - from dask.base import get_scheduler - actual_get = get_scheduler(get, collection) - except ImportError: - try: - from dask.utils import effective_get - actual_get = effective_get(get, collection) - except ImportError: - return None - - try: - from dask.distributed import Client - if isinstance(actual_get.__self__, Client): - return 'distributed' - except (ImportError, AttributeError): - try: - import dask.multiprocessing - if actual_get == dask.multiprocessing.get: - return 'multiprocessing' - else: - return 'threaded' - except ImportError: - return 'threaded' - - -def _get_scheduler_lock(scheduler, path_or_file=None): - """ Get the appropriate lock for a certain situation based onthe dask - scheduler used. - - See Also - -------- - dask.utils.get_scheduler_lock - """ - - if scheduler == 'distributed': - from dask.distributed import Lock - return Lock(path_or_file) - elif scheduler == 'multiprocessing': - return multiprocessing.Lock() - elif scheduler == 'threaded': - from dask.utils import SerializableLock - return SerializableLock() - else: - return threading.Lock() - - def _encode_variable_name(name): if name is None: name = NONE_VAR_NAME @@ -133,39 +66,6 @@ def robust_getitem(array, key, catch=Exception, max_retries=6, time.sleep(1e-3 * next_delay) -class CombinedLock(object): - """A combination of multiple locks. - - Like a locked door, a CombinedLock is locked if any of its constituent - locks are locked. - """ - - def __init__(self, locks): - self.locks = tuple(set(locks)) # remove duplicates - - def acquire(self, *args): - return all(lock.acquire(*args) for lock in self.locks) - - def release(self, *args): - for lock in self.locks: - lock.release(*args) - - def __enter__(self): - for lock in self.locks: - lock.__enter__() - - def __exit__(self, *args): - for lock in self.locks: - lock.__exit__(*args) - - @property - def locked(self): - return any(lock.locked for lock in self.locks) - - def __repr__(self): - return "CombinedLock(%r)" % list(self.locks) - - class BackendArray(NdimSizeLenMixin, indexing.ExplicitlyIndexed): def __array__(self, dtype=None): @@ -174,9 +74,6 @@ def __array__(self, dtype=None): class AbstractDataStore(Mapping): - _autoclose = None - _ds = None - _isopen = False def __iter__(self): return iter(self.variables) @@ -259,7 +156,7 @@ def __exit__(self, exception_type, exception_value, traceback): class ArrayWriter(object): - def __init__(self, lock=HDF5_LOCK): + def __init__(self, lock=None): self.sources = [] self.targets = [] self.lock = lock @@ -274,6 +171,9 @@ def add(self, source, target): def sync(self, compute=True): if self.sources: import dask.array as da + # TODO: consider wrapping targets with dask.delayed, if this makes + # for any discernable difference in perforance, e.g., + # targets = [dask.delayed(t) for t in self.targets] delayed_store = da.store(self.sources, self.targets, lock=self.lock, compute=compute, flush=True) @@ -283,11 +183,6 @@ def sync(self, compute=True): class AbstractWritableDataStore(AbstractDataStore): - def __init__(self, writer=None, lock=HDF5_LOCK): - if writer is None: - writer = ArrayWriter(lock=lock) - self.writer = writer - self.delayed_store = None def encode(self, variables, attributes): """ @@ -329,12 +224,6 @@ def set_attribute(self, k, v): # pragma: no cover def set_variable(self, k, v): # pragma: no cover raise NotImplementedError - def sync(self, compute=True): - if self._isopen and self._autoclose: - # datastore will be reopened during write - self.close() - self.delayed_store = self.writer.sync(compute=compute) - def store_dataset(self, dataset): """ in stores, variables are all variables AND coordinates @@ -345,7 +234,7 @@ def store_dataset(self, dataset): self.store(dataset, dataset.attrs) def store(self, variables, attributes, check_encoding_set=frozenset(), - unlimited_dims=None): + writer=None, unlimited_dims=None): """ Top level method for putting data on this store, this method: - encodes variables/attributes @@ -361,16 +250,19 @@ def store(self, variables, attributes, check_encoding_set=frozenset(), check_encoding_set : list-like List of variables that should be checked for invalid encoding values + writer : ArrayWriter unlimited_dims : list-like List of dimension names that should be treated as unlimited dimensions. """ + if writer is None: + writer = ArrayWriter() variables, attributes = self.encode(variables, attributes) self.set_attributes(attributes) self.set_dimensions(variables, unlimited_dims=unlimited_dims) - self.set_variables(variables, check_encoding_set, + self.set_variables(variables, check_encoding_set, writer, unlimited_dims=unlimited_dims) def set_attributes(self, attributes): @@ -386,7 +278,7 @@ def set_attributes(self, attributes): for k, v in iteritems(attributes): self.set_attribute(k, v) - def set_variables(self, variables, check_encoding_set, + def set_variables(self, variables, check_encoding_set, writer, unlimited_dims=None): """ This provides a centralized method to set the variables on the data @@ -399,6 +291,7 @@ def set_variables(self, variables, check_encoding_set, check_encoding_set : list-like List of variables that should be checked for invalid encoding values + writer : ArrayWriter unlimited_dims : list-like List of dimension names that should be treated as unlimited dimensions. @@ -410,7 +303,7 @@ def set_variables(self, variables, check_encoding_set, target, source = self.prepare_variable( name, v, check, unlimited_dims=unlimited_dims) - self.writer.add(source, target) + writer.add(source, target) def set_dimensions(self, variables, unlimited_dims=None): """ @@ -457,87 +350,3 @@ def encode(self, variables, attributes): attributes = OrderedDict([(k, self.encode_attribute(v)) for k, v in attributes.items()]) return variables, attributes - - -class DataStorePickleMixin(object): - """Subclasses must define `ds`, `_opener` and `_mode` attributes. - - Do not subclass this class: it is not part of xarray's external API. - """ - - def __getstate__(self): - state = self.__dict__.copy() - del state['_ds'] - del state['_isopen'] - if self._mode == 'w': - # file has already been created, don't override when restoring - state['_mode'] = 'a' - return state - - def __setstate__(self, state): - self.__dict__.update(state) - self._ds = None - self._isopen = False - - @property - def ds(self): - if self._ds is not None and self._isopen: - return self._ds - ds = self._opener(mode=self._mode) - self._isopen = True - return ds - - @contextlib.contextmanager - def ensure_open(self, autoclose=None): - """ - Helper function to make sure datasets are closed and opened - at appropriate times to avoid too many open file errors. - - Use requires `autoclose=True` argument to `open_mfdataset`. - """ - - if autoclose is None: - autoclose = self._autoclose - - if not self._isopen: - try: - self._ds = self._opener() - self._isopen = True - yield - finally: - if autoclose: - self.close() - else: - yield - - def assert_open(self): - if not self._isopen: - raise AssertionError('internal failure: file must be open ' - 'if `autoclose=True` is used.') - - -class PickleByReconstructionWrapper(object): - - def __init__(self, opener, file, mode='r', **kwargs): - self.opener = partial(opener, file, mode=mode, **kwargs) - self.mode = mode - self._ds = None - - @property - def value(self): - self._ds = self.opener() - return self._ds - - def __getstate__(self): - state = self.__dict__.copy() - del state['_ds'] - if self.mode == 'w': - # file has already been created, don't override when restoring - state['mode'] = 'a' - return state - - def __setstate__(self, state): - self.__dict__.update(state) - - def close(self): - self._ds.close() diff --git a/xarray/backends/file_manager.py b/xarray/backends/file_manager.py new file mode 100644 index 00000000000..a93285370b2 --- /dev/null +++ b/xarray/backends/file_manager.py @@ -0,0 +1,206 @@ +import threading + +from ..core import utils +from ..core.options import OPTIONS +from .lru_cache import LRUCache + + +# Global cache for storing open files. +FILE_CACHE = LRUCache( + OPTIONS['file_cache_maxsize'], on_evict=lambda k, v: v.close()) +assert FILE_CACHE.maxsize, 'file cache must be at least size one' + + +_DEFAULT_MODE = utils.ReprObject('') + + +class FileManager(object): + """Manager for acquiring and closing a file object. + + Use FileManager subclasses (CachingFileManager in particular) on backend + storage classes to automatically handle issues related to keeping track of + many open files and transferring them between multiple processes. + """ + + def acquire(self): + """Acquire the file object from this manager.""" + raise NotImplementedError + + def close(self, needs_lock=True): + """Close the file object associated with this manager, if needed.""" + raise NotImplementedError + + +class CachingFileManager(FileManager): + """Wrapper for automatically opening and closing file objects. + + Unlike files, CachingFileManager objects can be safely pickled and passed + between processes. They should be explicitly closed to release resources, + but a per-process least-recently-used cache for open files ensures that you + can safely create arbitrarily large numbers of FileManager objects. + + Don't directly close files acquired from a FileManager. Instead, call + FileManager.close(), which ensures that closed files are removed from the + cache as well. + + Example usage: + + manager = FileManager(open, 'example.txt', mode='w') + f = manager.acquire() + f.write(...) + manager.close() # ensures file is closed + + Note that as long as previous files are still cached, acquiring a file + multiple times from the same FileManager is essentially free: + + f1 = manager.acquire() + f2 = manager.acquire() + assert f1 is f2 + + """ + + def __init__(self, opener, *args, **keywords): + """Initialize a FileManager. + + Parameters + ---------- + opener : callable + Function that when called like ``opener(*args, **kwargs)`` returns + an open file object. The file object must implement a ``close()`` + method. + *args + Positional arguments for opener. A ``mode`` argument should be + provided as a keyword argument (see below). All arguments must be + hashable. + mode : optional + If provided, passed as a keyword argument to ``opener`` along with + ``**kwargs``. ``mode='w' `` has special treatment: after the first + call it is replaced by ``mode='a'`` in all subsequent function to + avoid overriding the newly created file. + kwargs : dict, optional + Keyword arguments for opener, excluding ``mode``. All values must + be hashable. + lock : duck-compatible threading.Lock, optional + Lock to use when modifying the cache inside acquire() and close(). + By default, uses a new threading.Lock() object. If set, this object + should be pickleable. + cache : MutableMapping, optional + Mapping to use as a cache for open files. By default, uses xarray's + global LRU file cache. Because ``cache`` typically points to a + global variable and contains non-picklable file objects, an + unpickled FileManager objects will be restored with the default + cache. + """ + # TODO: replace with real keyword arguments when we drop Python 2 + # support + mode = keywords.pop('mode', _DEFAULT_MODE) + kwargs = keywords.pop('kwargs', None) + lock = keywords.pop('lock', None) + cache = keywords.pop('cache', FILE_CACHE) + if keywords: + raise TypeError('FileManager() got unexpected keyword arguments: ' + '%s' % list(keywords)) + + self._opener = opener + self._args = args + self._mode = mode + self._kwargs = {} if kwargs is None else dict(kwargs) + self._default_lock = lock is None or lock is False + self._lock = threading.Lock() if self._default_lock else lock + self._cache = cache + self._key = self._make_key() + + def _make_key(self): + """Make a key for caching files in the LRU cache.""" + value = (self._opener, + self._args, + self._mode, + tuple(sorted(self._kwargs.items()))) + return _HashedSequence(value) + + def acquire(self): + """Acquiring a file object from the manager. + + A new file is only opened if it has expired from the + least-recently-used cache. + + This method uses a reentrant lock, which ensures that it is + thread-safe. You can safely acquire a file in multiple threads at the + same time, as long as the underlying file object is thread-safe. + + Returns + ------- + An open file object, as returned by ``opener(*args, **kwargs)``. + """ + with self._lock: + try: + file = self._cache[self._key] + except KeyError: + kwargs = self._kwargs + if self._mode is not _DEFAULT_MODE: + kwargs = kwargs.copy() + kwargs['mode'] = self._mode + file = self._opener(*self._args, **kwargs) + if self._mode == 'w': + # ensure file doesn't get overriden when opened again + self._mode = 'a' + self._key = self._make_key() + self._cache[self._key] = file + return file + + def _close(self): + default = None + file = self._cache.pop(self._key, default) + if file is not None: + file.close() + + def close(self, needs_lock=True): + """Explicitly close any associated file object (if necessary).""" + # TODO: remove needs_lock if/when we have a reentrant lock in + # dask.distributed: https://github.com/dask/dask/issues/3832 + if needs_lock: + with self._lock: + self._close() + else: + self._close() + + def __getstate__(self): + """State for pickling.""" + lock = None if self._default_lock else self._lock + return (self._opener, self._args, self._mode, self._kwargs, lock) + + def __setstate__(self, state): + """Restore from a pickle.""" + opener, args, mode, kwargs, lock = state + self.__init__(opener, *args, mode=mode, kwargs=kwargs, lock=lock) + + +class _HashedSequence(list): + """Speedup repeated look-ups by caching hash values. + + Based on what Python uses internally in functools.lru_cache. + + Python doesn't perform this optimization automatically: + https://bugs.python.org/issue1462796 + """ + + def __init__(self, tuple_value): + self[:] = tuple_value + self.hashvalue = hash(tuple_value) + + def __hash__(self): + return self.hashvalue + + +class DummyFileManager(FileManager): + """FileManager that simply wraps an open file in the FileManager interface. + """ + def __init__(self, value): + self._value = value + + def acquire(self): + return self._value + + def close(self, needs_lock=True): + del needs_lock # ignored + self._value.close() diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 959cd221734..59cd4e84793 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -8,11 +8,12 @@ from ..core import indexing from ..core.pycompat import OrderedDict, bytes_type, iteritems, unicode_type from ..core.utils import FrozenOrderedDict, close_on_error -from .common import ( - HDF5_LOCK, DataStorePickleMixin, WritableCFDataStore, find_root) +from .common import WritableCFDataStore +from .file_manager import CachingFileManager +from .locks import HDF5_LOCK, combine_locks, ensure_lock, get_write_lock from .netCDF4_ import ( - BaseNetCDF4Array, _encode_nc4_variable, _extract_nc4_variable_encoding, - _get_datatype, _nc4_require_group) + BaseNetCDF4Array, GroupWrapper, _encode_nc4_variable, + _extract_nc4_variable_encoding, _get_datatype, _nc4_require_group) class H5NetCDFArrayWrapper(BaseNetCDF4Array): @@ -25,8 +26,9 @@ def _getitem(self, key): # h5py requires using lists for fancy indexing: # https://github.com/h5py/h5py/issues/992 key = tuple(list(k) if isinstance(k, np.ndarray) else k for k in key) - with self.datastore.ensure_open(autoclose=True): - return self.get_array()[key] + array = self.get_array() + with self.datastore.lock: + return array[key] def maybe_decode_bytes(txt): @@ -61,104 +63,102 @@ def _open_h5netcdf_group(filename, mode, group): import h5netcdf ds = h5netcdf.File(filename, mode=mode) with close_on_error(ds): - return _nc4_require_group( + ds = _nc4_require_group( ds, group, mode, create_group=_h5netcdf_create_group) + return GroupWrapper(ds) -class H5NetCDFStore(WritableCFDataStore, DataStorePickleMixin): +class H5NetCDFStore(WritableCFDataStore): """Store for reading and writing data via h5netcdf """ def __init__(self, filename, mode='r', format=None, group=None, - writer=None, autoclose=False, lock=HDF5_LOCK): + lock=None, autoclose=False): if format not in [None, 'NETCDF4']: raise ValueError('invalid format for h5netcdf backend') - opener = functools.partial(_open_h5netcdf_group, filename, mode=mode, - group=group) - self._ds = opener() - if autoclose: - raise NotImplementedError('autoclose=True is not implemented ' - 'for the h5netcdf backend pending ' - 'further exploration, e.g., bug fixes ' - '(in h5netcdf?)') - self._autoclose = False - self._isopen = True + self._manager = CachingFileManager( + _open_h5netcdf_group, filename, mode=mode, + kwargs=dict(group=group)) + + if lock is None: + if mode == 'r': + lock = HDF5_LOCK + else: + lock = combine_locks([HDF5_LOCK, get_write_lock(filename)]) + self.format = format - self._opener = opener self._filename = filename self._mode = mode - super(H5NetCDFStore, self).__init__(writer, lock=lock) + self.lock = ensure_lock(lock) + self.autoclose = autoclose + + @property + def ds(self): + return self._manager.acquire().value def open_store_variable(self, name, var): import h5py - with self.ensure_open(autoclose=False): - dimensions = var.dimensions - data = indexing.LazilyOuterIndexedArray( - H5NetCDFArrayWrapper(name, self)) - attrs = _read_attributes(var) - - # netCDF4 specific encoding - encoding = { - 'chunksizes': var.chunks, - 'fletcher32': var.fletcher32, - 'shuffle': var.shuffle, - } - # Convert h5py-style compression options to NetCDF4-Python - # style, if possible - if var.compression == 'gzip': - encoding['zlib'] = True - encoding['complevel'] = var.compression_opts - elif var.compression is not None: - encoding['compression'] = var.compression - encoding['compression_opts'] = var.compression_opts - - # save source so __repr__ can detect if it's local or not - encoding['source'] = self._filename - encoding['original_shape'] = var.shape - - vlen_dtype = h5py.check_dtype(vlen=var.dtype) - if vlen_dtype is unicode_type: - encoding['dtype'] = str - elif vlen_dtype is not None: # pragma: no cover - # xarray doesn't support writing arbitrary vlen dtypes yet. - pass - else: - encoding['dtype'] = var.dtype + dimensions = var.dimensions + data = indexing.LazilyOuterIndexedArray( + H5NetCDFArrayWrapper(name, self)) + attrs = _read_attributes(var) + + # netCDF4 specific encoding + encoding = { + 'chunksizes': var.chunks, + 'fletcher32': var.fletcher32, + 'shuffle': var.shuffle, + } + # Convert h5py-style compression options to NetCDF4-Python + # style, if possible + if var.compression == 'gzip': + encoding['zlib'] = True + encoding['complevel'] = var.compression_opts + elif var.compression is not None: + encoding['compression'] = var.compression + encoding['compression_opts'] = var.compression_opts + + # save source so __repr__ can detect if it's local or not + encoding['source'] = self._filename + encoding['original_shape'] = var.shape + + vlen_dtype = h5py.check_dtype(vlen=var.dtype) + if vlen_dtype is unicode_type: + encoding['dtype'] = str + elif vlen_dtype is not None: # pragma: no cover + # xarray doesn't support writing arbitrary vlen dtypes yet. + pass + else: + encoding['dtype'] = var.dtype return Variable(dimensions, data, attrs, encoding) def get_variables(self): - with self.ensure_open(autoclose=False): - return FrozenOrderedDict((k, self.open_store_variable(k, v)) - for k, v in iteritems(self.ds.variables)) + return FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in iteritems(self.ds.variables)) def get_attrs(self): - with self.ensure_open(autoclose=True): - return FrozenOrderedDict(_read_attributes(self.ds)) + return FrozenOrderedDict(_read_attributes(self.ds)) def get_dimensions(self): - with self.ensure_open(autoclose=True): - return self.ds.dimensions + return self.ds.dimensions def get_encoding(self): - with self.ensure_open(autoclose=True): - encoding = {} - encoding['unlimited_dims'] = { - k for k, v in self.ds.dimensions.items() if v is None} + encoding = {} + encoding['unlimited_dims'] = { + k for k, v in self.ds.dimensions.items() if v is None} return encoding def set_dimension(self, name, length, is_unlimited=False): - with self.ensure_open(autoclose=False): - if is_unlimited: - self.ds.dimensions[name] = None - self.ds.resize_dimension(name, length) - else: - self.ds.dimensions[name] = length + if is_unlimited: + self.ds.dimensions[name] = None + self.ds.resize_dimension(name, length) + else: + self.ds.dimensions[name] = length def set_attribute(self, key, value): - with self.ensure_open(autoclose=False): - self.ds.attrs[key] = value + self.ds.attrs[key] = value def encode_variable(self, variable): return _encode_nc4_variable(variable) @@ -226,18 +226,11 @@ def prepare_variable(self, name, variable, check_encoding=False, return target, variable.data - def sync(self, compute=True): - if not compute: - raise NotImplementedError( - 'compute=False is not supported for the h5netcdf backend yet') - with self.ensure_open(autoclose=True): - super(H5NetCDFStore, self).sync(compute=compute) - self.ds.sync() - - def close(self): - if self._isopen: - # netCDF4 only allows closing the root group - ds = find_root(self.ds) - if not ds._closed: - ds.close() - self._isopen = False + def sync(self): + self.ds.sync() + # if self.autoclose: + # self.close() + # super(H5NetCDFStore, self).sync(compute=compute) + + def close(self, **kwargs): + self._manager.close(**kwargs) diff --git a/xarray/backends/locks.py b/xarray/backends/locks.py new file mode 100644 index 00000000000..f633280ef1d --- /dev/null +++ b/xarray/backends/locks.py @@ -0,0 +1,191 @@ +import multiprocessing +import threading +import weakref + +try: + from dask.utils import SerializableLock +except ImportError: + # no need to worry about serializing the lock + SerializableLock = threading.Lock + + +# Locks used by multiple backends. +# Neither HDF5 nor the netCDF-C library are thread-safe. +HDF5_LOCK = SerializableLock() +NETCDFC_LOCK = SerializableLock() + + +_FILE_LOCKS = weakref.WeakValueDictionary() + + +def _get_threaded_lock(key): + try: + lock = _FILE_LOCKS[key] + except KeyError: + lock = _FILE_LOCKS[key] = threading.Lock() + return lock + + +def _get_multiprocessing_lock(key): + # TODO: make use of the key -- maybe use locket.py? + # https://github.com/mwilliamson/locket.py + del key # unused + return multiprocessing.Lock() + + +def _get_distributed_lock(key): + from dask.distributed import Lock + return Lock(key) + + +_LOCK_MAKERS = { + None: _get_threaded_lock, + 'threaded': _get_threaded_lock, + 'multiprocessing': _get_multiprocessing_lock, + 'distributed': _get_distributed_lock, +} + + +def _get_lock_maker(scheduler=None): + """Returns an appropriate function for creating resource locks. + + Parameters + ---------- + scheduler : str or None + Dask scheduler being used. + + See Also + -------- + dask.utils.get_scheduler_lock + """ + return _LOCK_MAKERS[scheduler] + + +def _get_scheduler(get=None, collection=None): + """Determine the dask scheduler that is being used. + + None is returned if no dask scheduler is active. + + See also + -------- + dask.base.get_scheduler + """ + try: + # dask 0.18.1 and later + from dask.base import get_scheduler + actual_get = get_scheduler(get, collection) + except ImportError: + try: + from dask.utils import effective_get + actual_get = effective_get(get, collection) + except ImportError: + return None + + try: + from dask.distributed import Client + if isinstance(actual_get.__self__, Client): + return 'distributed' + except (ImportError, AttributeError): + try: + import dask.multiprocessing + if actual_get == dask.multiprocessing.get: + return 'multiprocessing' + else: + return 'threaded' + except ImportError: + return 'threaded' + + +def get_write_lock(key): + """Get a scheduler appropriate lock for writing to the given resource. + + Parameters + ---------- + key : str + Name of the resource for which to acquire a lock. Typically a filename. + + Returns + ------- + Lock object that can be used like a threading.Lock object. + """ + scheduler = _get_scheduler() + lock_maker = _get_lock_maker(scheduler) + return lock_maker(key) + + +class CombinedLock(object): + """A combination of multiple locks. + + Like a locked door, a CombinedLock is locked if any of its constituent + locks are locked. + """ + + def __init__(self, locks): + self.locks = tuple(set(locks)) # remove duplicates + + def acquire(self, *args): + return all(lock.acquire(*args) for lock in self.locks) + + def release(self, *args): + for lock in self.locks: + lock.release(*args) + + def __enter__(self): + for lock in self.locks: + lock.__enter__() + + def __exit__(self, *args): + for lock in self.locks: + lock.__exit__(*args) + + @property + def locked(self): + return any(lock.locked for lock in self.locks) + + def __repr__(self): + return "CombinedLock(%r)" % list(self.locks) + + +class DummyLock(object): + """DummyLock provides the lock API without any actual locking.""" + + def acquire(self, *args): + pass + + def release(self, *args): + pass + + def __enter__(self): + pass + + def __exit__(self, *args): + pass + + @property + def locked(self): + return False + + +def combine_locks(locks): + """Combine a sequence of locks into a single lock.""" + all_locks = [] + for lock in locks: + if isinstance(lock, CombinedLock): + all_locks.extend(lock.locks) + elif lock is not None: + all_locks.append(lock) + + num_locks = len(all_locks) + if num_locks > 1: + return CombinedLock(all_locks) + elif num_locks == 1: + return all_locks[0] + else: + return DummyLock() + + +def ensure_lock(lock): + """Ensure that the given object is a lock.""" + if lock is None or lock is False: + return DummyLock() + return lock diff --git a/xarray/backends/lru_cache.py b/xarray/backends/lru_cache.py new file mode 100644 index 00000000000..321a1ca4da4 --- /dev/null +++ b/xarray/backends/lru_cache.py @@ -0,0 +1,91 @@ +import collections +import threading + +from ..core.pycompat import move_to_end + + +class LRUCache(collections.MutableMapping): + """Thread-safe LRUCache based on an OrderedDict. + + All dict operations (__getitem__, __setitem__, __contains__) update the + priority of the relevant key and take O(1) time. The dict is iterated over + in order from the oldest to newest key, which means that a complete pass + over the dict should not affect the order of any entries. + + When a new item is set and the maximum size of the cache is exceeded, the + oldest item is dropped and called with ``on_evict(key, value)``. + + The ``maxsize`` property can be used to view or adjust the capacity of + the cache, e.g., ``cache.maxsize = new_size``. + """ + def __init__(self, maxsize, on_evict=None): + """ + Parameters + ---------- + maxsize : int + Integer maximum number of items to hold in the cache. + on_evict: callable, optional + Function to call like ``on_evict(key, value)`` when items are + evicted. + """ + if not isinstance(maxsize, int): + raise TypeError('maxsize must be an integer') + if maxsize < 0: + raise ValueError('maxsize must be non-negative') + self._maxsize = maxsize + self._on_evict = on_evict + self._cache = collections.OrderedDict() + self._lock = threading.RLock() + + def __getitem__(self, key): + # record recent use of the key by moving it to the front of the list + with self._lock: + value = self._cache[key] + move_to_end(self._cache, key) + return value + + def _enforce_size_limit(self, capacity): + """Shrink the cache if necessary, evicting the oldest items.""" + while len(self._cache) > capacity: + key, value = self._cache.popitem(last=False) + if self._on_evict is not None: + self._on_evict(key, value) + + def __setitem__(self, key, value): + with self._lock: + if key in self._cache: + # insert the new value at the end + del self._cache[key] + self._cache[key] = value + elif self._maxsize: + # make room if necessary + self._enforce_size_limit(self._maxsize - 1) + self._cache[key] = value + elif self._on_evict is not None: + # not saving, immediately evict + self._on_evict(key, value) + + def __delitem__(self, key): + del self._cache[key] + + def __iter__(self): + # create a list, so accessing the cache during iteration cannot change + # the iteration order + return iter(list(self._cache)) + + def __len__(self): + return len(self._cache) + + @property + def maxsize(self): + """Maximum number of items can be held in the cache.""" + return self._maxsize + + @maxsize.setter + def maxsize(self, size): + """Resize the cache, evicting the oldest items if necessary.""" + if size < 0: + raise ValueError('maxsize must be non-negative') + with self._lock: + self._enforce_size_limit(size) + self._maxsize = size diff --git a/xarray/backends/memory.py b/xarray/backends/memory.py index dcf092557b8..195d4647534 100644 --- a/xarray/backends/memory.py +++ b/xarray/backends/memory.py @@ -17,10 +17,9 @@ class InMemoryDataStore(AbstractWritableDataStore): This store exists purely for internal testing purposes. """ - def __init__(self, variables=None, attributes=None, writer=None): + def __init__(self, variables=None, attributes=None): self._variables = OrderedDict() if variables is None else variables self._attributes = OrderedDict() if attributes is None else attributes - super(InMemoryDataStore, self).__init__(writer) def get_attrs(self): return self._attributes diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index aa19633020b..08ba085b77e 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -13,8 +13,10 @@ from ..core.pycompat import PY3, OrderedDict, basestring, iteritems, suppress from ..core.utils import FrozenOrderedDict, close_on_error, is_remote_uri from .common import ( - HDF5_LOCK, BackendArray, DataStorePickleMixin, WritableCFDataStore, - find_root, robust_getitem) + BackendArray, WritableCFDataStore, find_root, robust_getitem) +from .locks import (NETCDFC_LOCK, HDF5_LOCK, + combine_locks, ensure_lock, get_write_lock) +from .file_manager import CachingFileManager, DummyFileManager from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable # This lookup table maps from dtype.byteorder to a readable endian @@ -25,6 +27,9 @@ '|': 'native'} +NETCDF4_PYTHON_LOCK = combine_locks([NETCDFC_LOCK, HDF5_LOCK]) + + class BaseNetCDF4Array(BackendArray): def __init__(self, variable_name, datastore): self.datastore = datastore @@ -42,12 +47,13 @@ def __init__(self, variable_name, datastore): self.dtype = dtype def __setitem__(self, key, value): - with self.datastore.ensure_open(autoclose=True): + with self.datastore.lock: data = self.get_array() data[key] = value + if self.datastore.autoclose: + self.datastore.close(needs_lock=False) def get_array(self): - self.datastore.assert_open() return self.datastore.ds.variables[self.variable_name] @@ -63,20 +69,22 @@ def _getitem(self, key): else: getitem = operator.getitem - with self.datastore.ensure_open(autoclose=True): - try: - array = getitem(self.get_array(), key) - except IndexError: - # Catch IndexError in netCDF4 and return a more informative - # error message. This is most often called when an unsorted - # indexer is used before the data is loaded from disk. - msg = ('The indexing operation you are attempting to perform ' - 'is not valid on netCDF4.Variable object. Try loading ' - 'your data into memory first by calling .load().') - if not PY3: - import traceback - msg += '\n\nOriginal traceback:\n' + traceback.format_exc() - raise IndexError(msg) + original_array = self.get_array() + + try: + with self.datastore.lock: + array = getitem(original_array, key) + except IndexError: + # Catch IndexError in netCDF4 and return a more informative + # error message. This is most often called when an unsorted + # indexer is used before the data is loaded from disk. + msg = ('The indexing operation you are attempting to perform ' + 'is not valid on netCDF4.Variable object. Try loading ' + 'your data into memory first by calling .load().') + if not PY3: + import traceback + msg += '\n\nOriginal traceback:\n' + traceback.format_exc() + raise IndexError(msg) return array @@ -223,7 +231,17 @@ def _extract_nc4_variable_encoding(variable, raise_on_invalid=False, return encoding -def _open_netcdf4_group(filename, mode, group=None, **kwargs): +class GroupWrapper(object): + """Wrap netCDF4.Group objects so closing them closes the root group.""" + def __init__(self, value): + self.value = value + + def close(self): + # netCDF4 only allows closing the root group + find_root(self.value).close() + + +def _open_netcdf4_group(filename, lock, mode, group=None, **kwargs): import netCDF4 as nc4 ds = nc4.Dataset(filename, mode=mode, **kwargs) @@ -233,7 +251,7 @@ def _open_netcdf4_group(filename, mode, group=None, **kwargs): _disable_auto_decode_group(ds) - return ds + return GroupWrapper(ds) def _disable_auto_decode_variable(var): @@ -279,40 +297,33 @@ def _set_nc_attribute(obj, key, value): obj.setncattr(key, value) -class NetCDF4DataStore(WritableCFDataStore, DataStorePickleMixin): +class NetCDF4DataStore(WritableCFDataStore): """Store for reading and writing data via the Python-NetCDF4 library. This store supports NetCDF3, NetCDF4 and OpenDAP datasets. """ - def __init__(self, netcdf4_dataset, mode='r', writer=None, opener=None, - autoclose=False, lock=HDF5_LOCK): - - if autoclose and opener is None: - raise ValueError('autoclose requires an opener') + def __init__(self, manager, lock=NETCDF4_PYTHON_LOCK, autoclose=False): + import netCDF4 - _disable_auto_decode_group(netcdf4_dataset) + if isinstance(manager, netCDF4.Dataset): + _disable_auto_decode_group(manager) + manager = DummyFileManager(GroupWrapper(manager)) - self._ds = netcdf4_dataset - self._autoclose = autoclose - self._isopen = True + self._manager = manager self.format = self.ds.data_model self._filename = self.ds.filepath() self.is_remote = is_remote_uri(self._filename) - self._mode = mode = 'a' if mode == 'w' else mode - if opener: - self._opener = functools.partial(opener, mode=self._mode) - else: - self._opener = opener - super(NetCDF4DataStore, self).__init__(writer, lock=lock) + self.lock = ensure_lock(lock) + self.autoclose = autoclose @classmethod def open(cls, filename, mode='r', format='NETCDF4', group=None, - writer=None, clobber=True, diskless=False, persist=False, - autoclose=False, lock=HDF5_LOCK): - import netCDF4 as nc4 + clobber=True, diskless=False, persist=False, + lock=None, lock_maker=None, autoclose=False): + import netCDF4 if (len(filename) == 88 and - LooseVersion(nc4.__version__) < "1.3.1"): + LooseVersion(netCDF4.__version__) < "1.3.1"): warnings.warn( 'A segmentation fault may occur when the ' 'file path has exactly 88 characters as it does ' @@ -323,86 +334,91 @@ def open(cls, filename, mode='r', format='NETCDF4', group=None, 'https://github.com/pydata/xarray/issues/1745') if format is None: format = 'NETCDF4' - opener = functools.partial(_open_netcdf4_group, filename, mode=mode, - group=group, clobber=clobber, - diskless=diskless, persist=persist, - format=format) - ds = opener() - return cls(ds, mode=mode, writer=writer, opener=opener, - autoclose=autoclose, lock=lock) - def open_store_variable(self, name, var): - with self.ensure_open(autoclose=False): - dimensions = var.dimensions - data = indexing.LazilyOuterIndexedArray( - NetCDF4ArrayWrapper(name, self)) - attributes = OrderedDict((k, var.getncattr(k)) - for k in var.ncattrs()) - _ensure_fill_value_valid(data, attributes) - # netCDF4 specific encoding; save _FillValue for later - encoding = {} - filters = var.filters() - if filters is not None: - encoding.update(filters) - chunking = var.chunking() - if chunking is not None: - if chunking == 'contiguous': - encoding['contiguous'] = True - encoding['chunksizes'] = None + if lock is None: + if mode == 'r': + if is_remote_uri(filename): + lock = NETCDFC_LOCK + else: + lock = NETCDF4_PYTHON_LOCK + else: + if format is None or format.startswith('NETCDF4'): + base_lock = NETCDF4_PYTHON_LOCK else: - encoding['contiguous'] = False - encoding['chunksizes'] = tuple(chunking) - # TODO: figure out how to round-trip "endian-ness" without raising - # warnings from netCDF4 - # encoding['endian'] = var.endian() - pop_to(attributes, encoding, 'least_significant_digit') - # save source so __repr__ can detect if it's local or not - encoding['source'] = self._filename - encoding['original_shape'] = var.shape - encoding['dtype'] = var.dtype + base_lock = NETCDFC_LOCK + lock = combine_locks([base_lock, get_write_lock(filename)]) + + manager = CachingFileManager( + _open_netcdf4_group, filename, lock, mode=mode, + kwargs=dict(group=group, clobber=clobber, diskless=diskless, + persist=persist, format=format)) + return cls(manager, lock=lock, autoclose=autoclose) + + @property + def ds(self): + return self._manager.acquire().value + + def open_store_variable(self, name, var): + dimensions = var.dimensions + data = indexing.LazilyOuterIndexedArray( + NetCDF4ArrayWrapper(name, self)) + attributes = OrderedDict((k, var.getncattr(k)) + for k in var.ncattrs()) + _ensure_fill_value_valid(data, attributes) + # netCDF4 specific encoding; save _FillValue for later + encoding = {} + filters = var.filters() + if filters is not None: + encoding.update(filters) + chunking = var.chunking() + if chunking is not None: + if chunking == 'contiguous': + encoding['contiguous'] = True + encoding['chunksizes'] = None + else: + encoding['contiguous'] = False + encoding['chunksizes'] = tuple(chunking) + # TODO: figure out how to round-trip "endian-ness" without raising + # warnings from netCDF4 + # encoding['endian'] = var.endian() + pop_to(attributes, encoding, 'least_significant_digit') + # save source so __repr__ can detect if it's local or not + encoding['source'] = self._filename + encoding['original_shape'] = var.shape + encoding['dtype'] = var.dtype return Variable(dimensions, data, attributes, encoding) def get_variables(self): - with self.ensure_open(autoclose=False): - dsvars = FrozenOrderedDict((k, self.open_store_variable(k, v)) - for k, v in - iteritems(self.ds.variables)) + dsvars = FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in + iteritems(self.ds.variables)) return dsvars def get_attrs(self): - with self.ensure_open(autoclose=True): - attrs = FrozenOrderedDict((k, self.ds.getncattr(k)) - for k in self.ds.ncattrs()) + attrs = FrozenOrderedDict((k, self.ds.getncattr(k)) + for k in self.ds.ncattrs()) return attrs def get_dimensions(self): - with self.ensure_open(autoclose=True): - dims = FrozenOrderedDict((k, len(v)) - for k, v in iteritems(self.ds.dimensions)) + dims = FrozenOrderedDict((k, len(v)) + for k, v in iteritems(self.ds.dimensions)) return dims def get_encoding(self): - with self.ensure_open(autoclose=True): - encoding = {} - encoding['unlimited_dims'] = { - k for k, v in self.ds.dimensions.items() if v.isunlimited()} + encoding = {} + encoding['unlimited_dims'] = { + k for k, v in self.ds.dimensions.items() if v.isunlimited()} return encoding def set_dimension(self, name, length, is_unlimited=False): - with self.ensure_open(autoclose=False): - dim_length = length if not is_unlimited else None - self.ds.createDimension(name, size=dim_length) + dim_length = length if not is_unlimited else None + self.ds.createDimension(name, size=dim_length) def set_attribute(self, key, value): - with self.ensure_open(autoclose=False): - if self.format != 'NETCDF4': - value = encode_nc3_attr_value(value) - _set_nc_attribute(self.ds, key, value) - - def set_variables(self, *args, **kwargs): - with self.ensure_open(autoclose=False): - super(NetCDF4DataStore, self).set_variables(*args, **kwargs) + if self.format != 'NETCDF4': + value = encode_nc3_attr_value(value) + _set_nc_attribute(self.ds, key, value) def encode_variable(self, variable): variable = _force_native_endianness(variable) @@ -460,15 +476,8 @@ def prepare_variable(self, name, variable, check_encoding=False, return target, variable.data - def sync(self, compute=True): - with self.ensure_open(autoclose=True): - super(NetCDF4DataStore, self).sync(compute=compute) - self.ds.sync() + def sync(self): + self.ds.sync() - def close(self): - if self._isopen: - # netCDF4 only allows closing the root group - ds = find_root(self.ds) - if ds._isopen: - ds.close() - self._isopen = False + def close(self, **kwargs): + self._manager.close(**kwargs) diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index 3d846916740..e4691d1f7e1 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -1,14 +1,18 @@ from __future__ import absolute_import, division, print_function -import functools - import numpy as np from .. import Variable from ..core import indexing from ..core.pycompat import OrderedDict -from ..core.utils import Frozen, FrozenOrderedDict -from .common import AbstractDataStore, BackendArray, DataStorePickleMixin +from ..core.utils import Frozen +from .common import AbstractDataStore, BackendArray +from .file_manager import CachingFileManager +from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock + + +# psuedonetcdf can invoke netCDF libraries internally +PNETCDF_LOCK = combine_locks([HDF5_LOCK, NETCDFC_LOCK]) class PncArrayWrapper(BackendArray): @@ -21,7 +25,6 @@ def __init__(self, variable_name, datastore): self.dtype = np.dtype(array.dtype) def get_array(self): - self.datastore.assert_open() return self.datastore.ds.variables[self.variable_name] def __getitem__(self, key): @@ -30,57 +33,55 @@ def __getitem__(self, key): self._getitem) def _getitem(self, key): - with self.datastore.ensure_open(autoclose=True): - return self.get_array()[key] + array = self.get_array() + with self.datastore.lock: + return array[key] -class PseudoNetCDFDataStore(AbstractDataStore, DataStorePickleMixin): +class PseudoNetCDFDataStore(AbstractDataStore): """Store for accessing datasets via PseudoNetCDF """ @classmethod - def open(cls, filename, format=None, writer=None, - autoclose=False, **format_kwds): + def open(cls, filename, lock=None, **format_kwds): from PseudoNetCDF import pncopen - opener = functools.partial(pncopen, filename, **format_kwds) - ds = opener() - mode = format_kwds.get('mode', 'r') - return cls(ds, mode=mode, writer=writer, opener=opener, - autoclose=autoclose) - def __init__(self, pnc_dataset, mode='r', writer=None, opener=None, - autoclose=False): + keywords = dict(kwargs=format_kwds) + # only include mode if explicitly passed + mode = format_kwds.pop('mode', None) + if mode is not None: + keywords['mode'] = mode + + if lock is None: + lock = PNETCDF_LOCK + + manager = CachingFileManager(pncopen, filename, lock=lock, **keywords) + return cls(manager, lock) - if autoclose and opener is None: - raise ValueError('autoclose requires an opener') + def __init__(self, manager, lock=None): + self._manager = manager + self.lock = ensure_lock(lock) - self._ds = pnc_dataset - self._autoclose = autoclose - self._isopen = True - self._opener = opener - self._mode = mode - super(PseudoNetCDFDataStore, self).__init__() + @property + def ds(self): + return self._manager.acquire() def open_store_variable(self, name, var): - with self.ensure_open(autoclose=False): - data = indexing.LazilyOuterIndexedArray( - PncArrayWrapper(name, self) - ) + data = indexing.LazilyOuterIndexedArray( + PncArrayWrapper(name, self) + ) attrs = OrderedDict((k, getattr(var, k)) for k in var.ncattrs()) return Variable(var.dimensions, data, attrs) def get_variables(self): - with self.ensure_open(autoclose=False): - return FrozenOrderedDict((k, self.open_store_variable(k, v)) - for k, v in self.ds.variables.items()) + return ((k, self.open_store_variable(k, v)) + for k, v in self.ds.variables.items()) def get_attrs(self): - with self.ensure_open(autoclose=True): - return Frozen(dict([(k, getattr(self.ds, k)) - for k in self.ds.ncattrs()])) + return Frozen(dict([(k, getattr(self.ds, k)) + for k in self.ds.ncattrs()])) def get_dimensions(self): - with self.ensure_open(autoclose=True): - return Frozen(self.ds.dimensions) + return Frozen(self.ds.dimensions) def get_encoding(self): encoding = {} @@ -90,6 +91,4 @@ def get_encoding(self): return encoding def close(self): - if self._isopen: - self.ds.close() - self._isopen = False + self._manager.close() diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 98b76928597..574fff744e3 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -1,13 +1,20 @@ from __future__ import absolute_import, division, print_function -import functools - import numpy as np from .. import Variable from ..core import indexing from ..core.utils import Frozen, FrozenOrderedDict -from .common import AbstractDataStore, BackendArray, DataStorePickleMixin +from .common import AbstractDataStore, BackendArray +from .file_manager import CachingFileManager +from .locks import ( + HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock, SerializableLock) + + +# PyNIO can invoke netCDF libraries internally +# Add a dedicated lock just in case NCL as well isn't thread-safe. +NCL_LOCK = SerializableLock() +PYNIO_LOCK = combine_locks([HDF5_LOCK, NETCDFC_LOCK, NCL_LOCK]) class NioArrayWrapper(BackendArray): @@ -20,7 +27,6 @@ def __init__(self, variable_name, datastore): self.dtype = np.dtype(array.typecode()) def get_array(self): - self.datastore.assert_open() return self.datastore.ds.variables[self.variable_name] def __getitem__(self, key): @@ -28,46 +34,45 @@ def __getitem__(self, key): key, self.shape, indexing.IndexingSupport.BASIC, self._getitem) def _getitem(self, key): - with self.datastore.ensure_open(autoclose=True): - array = self.get_array() + array = self.get_array() + with self.datastore.lock: if key == () and self.ndim == 0: return array.get_value() - return array[key] -class NioDataStore(AbstractDataStore, DataStorePickleMixin): +class NioDataStore(AbstractDataStore): """Store for accessing datasets via PyNIO """ - def __init__(self, filename, mode='r', autoclose=False): + def __init__(self, filename, mode='r', lock=None): import Nio - opener = functools.partial(Nio.open_file, filename, mode=mode) - self._ds = opener() - self._autoclose = autoclose - self._isopen = True - self._opener = opener - self._mode = mode + if lock is None: + lock = PYNIO_LOCK + self.lock = ensure_lock(lock) + self._manager = CachingFileManager( + Nio.open_file, filename, lock=lock, mode=mode) # xarray provides its own support for FillValue, # so turn off PyNIO's support for the same. self.ds.set_option('MaskedArrayMode', 'MaskedNever') + @property + def ds(self): + return self._manager.acquire() + def open_store_variable(self, name, var): data = indexing.LazilyOuterIndexedArray(NioArrayWrapper(name, self)) return Variable(var.dimensions, data, var.attributes) def get_variables(self): - with self.ensure_open(autoclose=False): - return FrozenOrderedDict((k, self.open_store_variable(k, v)) - for k, v in self.ds.variables.items()) + return FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in self.ds.variables.items()) def get_attrs(self): - with self.ensure_open(autoclose=True): - return Frozen(self.ds.attributes) + return Frozen(self.ds.attributes) def get_dimensions(self): - with self.ensure_open(autoclose=True): - return Frozen(self.ds.dimensions) + return Frozen(self.ds.dimensions) def get_encoding(self): encoding = {} @@ -76,6 +81,4 @@ def get_encoding(self): return encoding def close(self): - if self._isopen: - self.ds.close() - self._isopen = False + self._manager.close() diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 44cca9aaaf8..5746b4e748d 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -8,14 +8,13 @@ from .. import DataArray from ..core import indexing from ..core.utils import is_scalar -from .common import BackendArray, PickleByReconstructionWrapper +from .common import BackendArray +from .file_manager import CachingFileManager +from .locks import SerializableLock -try: - from dask.utils import SerializableLock as Lock -except ImportError: - from threading import Lock -RASTERIO_LOCK = Lock() +# TODO: should this be GDAL_LOCK instead? +RASTERIO_LOCK = SerializableLock() _ERROR_MSG = ('The kind of indexing operation you are trying to do is not ' 'valid on rasterio files. Try to load your data with ds.load()' @@ -25,18 +24,22 @@ class RasterioArrayWrapper(BackendArray): """A wrapper around rasterio dataset objects""" - def __init__(self, riods): - self.riods = riods - self._shape = (riods.value.count, riods.value.height, - riods.value.width) - self._ndims = len(self.shape) + def __init__(self, manager): + self.manager = manager - @property - def dtype(self): - dtypes = self.riods.value.dtypes + # cannot save riods as an attribute: this would break pickleability + riods = manager.acquire() + + self._shape = (riods.count, riods.height, riods.width) + + dtypes = riods.dtypes if not np.all(np.asarray(dtypes) == dtypes[0]): raise ValueError('All bands should have the same dtype') - return np.dtype(dtypes[0]) + self._dtype = np.dtype(dtypes[0]) + + @property + def dtype(self): + return self._dtype @property def shape(self): @@ -108,7 +111,8 @@ def _getitem(self, key): stop - start for (start, stop) in window) out = np.zeros(shape, dtype=self.dtype) else: - out = self.riods.value.read(band_key, window=window) + riods = self.manager.acquire() + out = riods.read(band_key, window=window) if squeeze_axis: out = np.squeeze(out, axis=squeeze_axis) @@ -203,7 +207,8 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, import rasterio - riods = PickleByReconstructionWrapper(rasterio.open, filename, mode='r') + manager = CachingFileManager(rasterio.open, filename, mode='r') + riods = manager.acquire() if cache is None: cache = chunks is None @@ -211,20 +216,20 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, coords = OrderedDict() # Get bands - if riods.value.count < 1: + if riods.count < 1: raise ValueError('Unknown dims') - coords['band'] = np.asarray(riods.value.indexes) + coords['band'] = np.asarray(riods.indexes) # Get coordinates if LooseVersion(rasterio.__version__) < '1.0': - transform = riods.value.affine + transform = riods.affine else: - transform = riods.value.transform + transform = riods.transform if transform.is_rectilinear: # 1d coordinates parse = True if parse_coordinates is None else parse_coordinates if parse: - nx, ny = riods.value.width, riods.value.height + nx, ny = riods.width, riods.height # xarray coordinates are pixel centered x, _ = (np.arange(nx) + 0.5, np.zeros(nx) + 0.5) * transform _, y = (np.zeros(ny) + 0.5, np.arange(ny) + 0.5) * transform @@ -234,57 +239,60 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, # 2d coordinates parse = False if (parse_coordinates is None) else parse_coordinates if parse: - warnings.warn("The file coordinates' transformation isn't " - "rectilinear: xarray won't parse the coordinates " - "in this case. Set `parse_coordinates=False` to " - "suppress this warning.", - RuntimeWarning, stacklevel=3) + warnings.warn( + "The file coordinates' transformation isn't " + "rectilinear: xarray won't parse the coordinates " + "in this case. Set `parse_coordinates=False` to " + "suppress this warning.", + RuntimeWarning, stacklevel=3) # Attributes attrs = dict() # Affine transformation matrix (always available) # This describes coefficients mapping pixel coordinates to CRS # For serialization store as tuple of 6 floats, the last row being - # always (0, 0, 1) per definition (see https://github.com/sgillies/affine) + # always (0, 0, 1) per definition (see + # https://github.com/sgillies/affine) attrs['transform'] = tuple(transform)[:6] - if hasattr(riods.value, 'crs') and riods.value.crs: + if hasattr(riods, 'crs') and riods.crs: # CRS is a dict-like object specific to rasterio # If CRS is not None, we convert it back to a PROJ4 string using # rasterio itself - attrs['crs'] = riods.value.crs.to_string() - if hasattr(riods.value, 'res'): + attrs['crs'] = riods.crs.to_string() + if hasattr(riods, 'res'): # (width, height) tuple of pixels in units of CRS - attrs['res'] = riods.value.res - if hasattr(riods.value, 'is_tiled'): + attrs['res'] = riods.res + if hasattr(riods, 'is_tiled'): # Is the TIF tiled? (bool) # We cast it to an int for netCDF compatibility - attrs['is_tiled'] = np.uint8(riods.value.is_tiled) - if hasattr(riods.value, 'nodatavals'): + attrs['is_tiled'] = np.uint8(riods.is_tiled) + if hasattr(riods, 'nodatavals'): # The nodata values for the raster bands - attrs['nodatavals'] = tuple([np.nan if nodataval is None else nodataval - for nodataval in riods.value.nodatavals]) + attrs['nodatavals'] = tuple( + np.nan if nodataval is None else nodataval + for nodataval in riods.nodatavals) # Parse extra metadata from tags, if supported parsers = {'ENVI': _parse_envi} - driver = riods.value.driver + driver = riods.driver if driver in parsers: - meta = parsers[driver](riods.value.tags(ns=driver)) + meta = parsers[driver](riods.tags(ns=driver)) for k, v in meta.items(): # Add values as coordinates if they match the band count, # as attributes otherwise if (isinstance(v, (list, np.ndarray)) and - len(v) == riods.value.count): + len(v) == riods.count): coords[k] = ('band', np.asarray(v)) else: attrs[k] = v - data = indexing.LazilyOuterIndexedArray(RasterioArrayWrapper(riods)) + data = indexing.LazilyOuterIndexedArray(RasterioArrayWrapper(manager)) # this lets you write arrays loaded with rasterio data = indexing.CopyOnWriteArray(data) - if cache and (chunks is None): + if cache and chunks is None: data = indexing.MemoryCachedArray(data) result = DataArray(data=data, dims=('band', 'y', 'x'), @@ -306,6 +314,6 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, lock=lock) # Make the file closeable - result._file_obj = riods + result._file_obj = manager return result diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index cd84431f6b7..b009342efb6 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -1,6 +1,5 @@ from __future__ import absolute_import, division, print_function -import functools import warnings from distutils.version import LooseVersion from io import BytesIO @@ -11,7 +10,9 @@ from ..core.indexing import NumpyIndexingAdapter from ..core.pycompat import OrderedDict, basestring, iteritems from ..core.utils import Frozen, FrozenOrderedDict -from .common import BackendArray, DataStorePickleMixin, WritableCFDataStore +from .common import BackendArray, WritableCFDataStore +from .locks import get_write_lock +from .file_manager import CachingFileManager, DummyFileManager from .netcdf3 import ( encode_nc3_attr_value, encode_nc3_variable, is_valid_nc3_name) @@ -40,31 +41,26 @@ def __init__(self, variable_name, datastore): str(array.dtype.itemsize)) def get_array(self): - self.datastore.assert_open() return self.datastore.ds.variables[self.variable_name].data def __getitem__(self, key): - with self.datastore.ensure_open(autoclose=True): - data = NumpyIndexingAdapter(self.get_array())[key] - # Copy data if the source file is mmapped. - # This makes things consistent - # with the netCDF4 library by ensuring - # we can safely read arrays even - # after closing associated files. - copy = self.datastore.ds.use_mmap - return np.array(data, dtype=self.dtype, copy=copy) + data = NumpyIndexingAdapter(self.get_array())[key] + # Copy data if the source file is mmapped. This makes things consistent + # with the netCDF4 library by ensuring we can safely read arrays even + # after closing associated files. + copy = self.datastore.ds.use_mmap + return np.array(data, dtype=self.dtype, copy=copy) def __setitem__(self, key, value): - with self.datastore.ensure_open(autoclose=True): - data = self.datastore.ds.variables[self.variable_name] - try: - data[key] = value - except TypeError: - if key is Ellipsis: - # workaround for GH: scipy/scipy#6880 - data[:] = value - else: - raise + data = self.datastore.ds.variables[self.variable_name] + try: + data[key] = value + except TypeError: + if key is Ellipsis: + # workaround for GH: scipy/scipy#6880 + data[:] = value + else: + raise def _open_scipy_netcdf(filename, mode, mmap, version): @@ -106,7 +102,7 @@ def _open_scipy_netcdf(filename, mode, mmap, version): raise -class ScipyDataStore(WritableCFDataStore, DataStorePickleMixin): +class ScipyDataStore(WritableCFDataStore): """Store for reading and writing data via scipy.io.netcdf. This store has the advantage of being able to be initialized with a @@ -116,7 +112,7 @@ class ScipyDataStore(WritableCFDataStore, DataStorePickleMixin): """ def __init__(self, filename_or_obj, mode='r', format=None, group=None, - writer=None, mmap=None, autoclose=False, lock=None): + mmap=None, lock=None): import scipy import scipy.io @@ -140,34 +136,38 @@ def __init__(self, filename_or_obj, mode='r', format=None, group=None, raise ValueError('invalid format for scipy.io.netcdf backend: %r' % format) - opener = functools.partial(_open_scipy_netcdf, - filename=filename_or_obj, - mode=mode, mmap=mmap, version=version) - self._ds = opener() - self._autoclose = autoclose - self._isopen = True - self._opener = opener - self._mode = mode + if (lock is None and mode != 'r' and + isinstance(filename_or_obj, basestring)): + lock = get_write_lock(filename_or_obj) + + if isinstance(filename_or_obj, basestring): + manager = CachingFileManager( + _open_scipy_netcdf, filename_or_obj, mode=mode, lock=lock, + kwargs=dict(mmap=mmap, version=version)) + else: + scipy_dataset = _open_scipy_netcdf( + filename_or_obj, mode=mode, mmap=mmap, version=version) + manager = DummyFileManager(scipy_dataset) + + self._manager = manager - super(ScipyDataStore, self).__init__(writer, lock=lock) + @property + def ds(self): + return self._manager.acquire() def open_store_variable(self, name, var): - with self.ensure_open(autoclose=False): - return Variable(var.dimensions, ScipyArrayWrapper(name, self), - _decode_attrs(var._attributes)) + return Variable(var.dimensions, ScipyArrayWrapper(name, self), + _decode_attrs(var._attributes)) def get_variables(self): - with self.ensure_open(autoclose=False): - return FrozenOrderedDict((k, self.open_store_variable(k, v)) - for k, v in iteritems(self.ds.variables)) + return FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in iteritems(self.ds.variables)) def get_attrs(self): - with self.ensure_open(autoclose=True): - return Frozen(_decode_attrs(self.ds._attributes)) + return Frozen(_decode_attrs(self.ds._attributes)) def get_dimensions(self): - with self.ensure_open(autoclose=True): - return Frozen(self.ds.dimensions) + return Frozen(self.ds.dimensions) def get_encoding(self): encoding = {} @@ -176,22 +176,20 @@ def get_encoding(self): return encoding def set_dimension(self, name, length, is_unlimited=False): - with self.ensure_open(autoclose=False): - if name in self.ds.dimensions: - raise ValueError('%s does not support modifying dimensions' - % type(self).__name__) - dim_length = length if not is_unlimited else None - self.ds.createDimension(name, dim_length) + if name in self.ds.dimensions: + raise ValueError('%s does not support modifying dimensions' + % type(self).__name__) + dim_length = length if not is_unlimited else None + self.ds.createDimension(name, dim_length) def _validate_attr_key(self, key): if not is_valid_nc3_name(key): raise ValueError("Not a valid attribute name") def set_attribute(self, key, value): - with self.ensure_open(autoclose=False): - self._validate_attr_key(key) - value = encode_nc3_attr_value(value) - setattr(self.ds, key, value) + self._validate_attr_key(key) + value = encode_nc3_attr_value(value) + setattr(self.ds, key, value) def encode_variable(self, variable): variable = encode_nc3_variable(variable) @@ -219,27 +217,8 @@ def prepare_variable(self, name, variable, check_encoding=False, return target, data - def sync(self, compute=True): - if not compute: - raise NotImplementedError( - 'compute=False is not supported for the scipy backend yet') - with self.ensure_open(autoclose=True): - super(ScipyDataStore, self).sync(compute=compute) - self.ds.flush() + def sync(self): + self.ds.sync() def close(self): - self.ds.close() - self._isopen = False - - def __exit__(self, type, value, tb): - self.close() - - def __setstate__(self, state): - filename = state['_opener'].keywords['filename'] - if hasattr(filename, 'seek'): - # it's a file-like object - # seek to the start of the file so scipy can read it - filename.seek(0) - super(ScipyDataStore, self).__setstate__(state) - self._ds = None - self._isopen = False + self._manager.close() diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 47b90c8a617..5f19c826289 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -217,8 +217,7 @@ class ZarrStore(AbstractWritableDataStore): """ @classmethod - def open_group(cls, store, mode='r', synchronizer=None, group=None, - writer=None): + def open_group(cls, store, mode='r', synchronizer=None, group=None): import zarr min_zarr = '2.2' @@ -230,24 +229,14 @@ def open_group(cls, store, mode='r', synchronizer=None, group=None, "#installation" % min_zarr) zarr_group = zarr.open_group(store=store, mode=mode, synchronizer=synchronizer, path=group) - return cls(zarr_group, writer=writer) + return cls(zarr_group) - def __init__(self, zarr_group, writer=None): + def __init__(self, zarr_group): self.ds = zarr_group self._read_only = self.ds.read_only self._synchronizer = self.ds.synchronizer self._group = self.ds.path - if writer is None: - # by default, we should not need a lock for writing zarr because - # we do not (yet) allow overlapping chunks during write - zarr_writer = ArrayWriter(lock=False) - else: - zarr_writer = writer - - # do we need to define attributes for all of the opener keyword args? - super(ZarrStore, self).__init__(zarr_writer) - def open_store_variable(self, name, zarr_array): data = indexing.LazilyOuterIndexedArray(ZarrArrayWrapper(name, self)) dimensions, attributes = _get_zarr_dims_and_attrs(zarr_array, @@ -334,8 +323,8 @@ def store(self, variables, attributes, *args, **kwargs): AbstractWritableDataStore.store(self, variables, attributes, *args, **kwargs) - def sync(self, compute=True): - self.delayed_store = self.writer.sync(compute=compute) + def sync(self): + pass def open_zarr(store, group=None, synchronizer=None, auto_chunk=True, diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 4ade15825c6..c8586d1d408 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1161,27 +1161,12 @@ def reset_coords(self, names=None, drop=False, inplace=False): del obj._variables[name] return obj - def dump_to_store(self, store, encoder=None, sync=True, encoding=None, - unlimited_dims=None, compute=True): + def dump_to_store(self, store, **kwargs): """Store dataset contents to a backends.*DataStore object.""" - if encoding is None: - encoding = {} - variables, attrs = conventions.encode_dataset_coordinates(self) - - check_encoding = set() - for k, enc in encoding.items(): - # no need to shallow copy the variable again; that already happened - # in encode_dataset_coordinates - variables[k].encoding = enc - check_encoding.add(k) - - if encoder: - variables, attrs = encoder(variables, attrs) - - store.store(variables, attrs, check_encoding, - unlimited_dims=unlimited_dims) - if sync: - store.sync(compute=compute) + from ..backends.api import dump_to_store + # TODO: rename and/or cleanup this method to make it more consistent + # with to_netcdf() + return dump_to_store(self, store, **kwargs) def to_netcdf(self, path=None, mode='w', format=None, group=None, engine=None, encoding=None, unlimited_dims=None, diff --git a/xarray/core/options.py b/xarray/core/options.py index a6118f02ed3..04ea0be7172 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -1,11 +1,43 @@ from __future__ import absolute_import, division, print_function +DISPLAY_WIDTH = 'display_width' +ARITHMETIC_JOIN = 'arithmetic_join' +ENABLE_CFTIMEINDEX = 'enable_cftimeindex' +FILE_CACHE_MAXSIZE = 'file_cache_maxsize' +CMAP_SEQUENTIAL = 'cmap_sequential' +CMAP_DIVERGENT = 'cmap_divergent' + OPTIONS = { - 'display_width': 80, - 'arithmetic_join': 'inner', - 'enable_cftimeindex': False, - 'cmap_sequential': 'viridis', - 'cmap_divergent': 'RdBu_r', + DISPLAY_WIDTH: 80, + ARITHMETIC_JOIN: 'inner', + ENABLE_CFTIMEINDEX: False, + FILE_CACHE_MAXSIZE: 128, + CMAP_SEQUENTIAL: 'viridis', + CMAP_DIVERGENT: 'RdBu_r', +} + +_JOIN_OPTIONS = frozenset(['inner', 'outer', 'left', 'right', 'exact']) + + +def _positive_integer(value): + return isinstance(value, int) and value > 0 + + +_VALIDATORS = { + DISPLAY_WIDTH: _positive_integer, + ARITHMETIC_JOIN: _JOIN_OPTIONS.__contains__, + ENABLE_CFTIMEINDEX: lambda value: isinstance(value, bool), + FILE_CACHE_MAXSIZE: _positive_integer, +} + + +def _set_file_cache_maxsize(value): + from ..backends.file_manager import FILE_CACHE + FILE_CACHE.maxsize = value + + +_SETTERS = { + FILE_CACHE_MAXSIZE: _set_file_cache_maxsize, } @@ -21,6 +53,10 @@ class set_options(object): - ``enable_cftimeindex``: flag to enable using a ``CFTimeIndex`` for time indexes with non-standard calendars or dates outside the Timestamp-valid range. Default: ``False``. + - ``file_cache_maxsize``: maximum number of open files to hold in xarray's + global least-recently-usage cached. This should be smaller than your + system's per-process file descriptor limit, e.g., ``ulimit -n`` on Linux. + Default: 128. - ``cmap_sequential``: colormap to use for nondivergent data plots. Default: ``viridis``. If string, must be matplotlib built-in colormap. Can also be a Colormap object (e.g. mpl.cm.magma) @@ -28,8 +64,7 @@ class set_options(object): Default: ``RdBu_r``. If string, must be matplotlib built-in colormap. Can also be a Colormap object (e.g. mpl.cm.magma) - - You can use ``set_options`` either as a context manager: +f You can use ``set_options`` either as a context manager: >>> ds = xr.Dataset({'x': np.arange(1000)}) >>> with xr.set_options(display_width=40): @@ -47,16 +82,26 @@ class set_options(object): """ def __init__(self, **kwargs): - invalid_options = {k for k in kwargs if k not in OPTIONS} - if invalid_options: - raise ValueError('argument names %r are not in the set of valid ' - 'options %r' % (invalid_options, set(OPTIONS))) self.old = OPTIONS.copy() - OPTIONS.update(kwargs) + for k, v in kwargs.items(): + if k not in OPTIONS: + raise ValueError( + 'argument name %r is not in the set of valid options %r' + % (k, set(OPTIONS))) + if k in _VALIDATORS and not _VALIDATORS[k](v): + raise ValueError( + 'option %r given an invalid value: %r' % (k, v)) + self._apply_update(kwargs) + + def _apply_update(self, options_dict): + for k, v in options_dict.items(): + if k in _SETTERS: + _SETTERS[k](v) + OPTIONS.update(options_dict) def __enter__(self): return def __exit__(self, type, value, traceback): OPTIONS.clear() - OPTIONS.update(self.old) + self._apply_update(self.old) diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 78c26f1e92f..b980bc279b0 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -28,6 +28,9 @@ def itervalues(d): import builtins from urllib.request import urlretrieve from inspect import getfullargspec as getargspec + + def move_to_end(ordered_dict, key): + ordered_dict.move_to_end(key) else: # pragma: no cover # Python 2 basestring = basestring # noqa @@ -50,6 +53,11 @@ def itervalues(d): from urllib import urlretrieve from inspect import getargspec + def move_to_end(ordered_dict, key): + value = ordered_dict[key] + del ordered_dict[key] + ordered_dict[key] = value + integer_types = native_int_types + (np.integer,) try: @@ -76,7 +84,6 @@ def itervalues(d): except ImportError as e: path_type = () - try: from contextlib import suppress except ImportError: diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 0d97ed70fa3..43811942d5f 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -8,7 +8,6 @@ import shutil import sys import tempfile -import unittest import warnings from io import BytesIO @@ -20,13 +19,13 @@ from xarray import ( DataArray, Dataset, backends, open_dataarray, open_dataset, open_mfdataset, save_mfdataset) -from xarray.backends.common import ( - PickleByReconstructionWrapper, robust_getitem) +from xarray.backends.common import robust_getitem from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.backends.pydap_ import PydapDataStore from xarray.core import indexing from xarray.core.pycompat import ( - PY2, ExitStack, basestring, dask_array_type, iteritems) + ExitStack, basestring, dask_array_type, iteritems) +from xarray.core.options import set_options from xarray.tests import mock from . import ( @@ -138,7 +137,6 @@ class NetCDF3Only(object): class DatasetIOTestCases(object): - autoclose = False engine = None file_format = None @@ -172,8 +170,7 @@ def save(self, dataset, path, **kwargs): @contextlib.contextmanager def open(self, path, **kwargs): - with open_dataset(path, engine=self.engine, autoclose=self.autoclose, - **kwargs) as ds: + with open_dataset(path, engine=self.engine, **kwargs) as ds: yield ds def test_zero_dimensional_variable(self): @@ -1159,10 +1156,10 @@ def test_already_open_dataset(self): v[...] = 42 nc = nc4.Dataset(tmp_file, mode='r') - with backends.NetCDF4DataStore(nc, autoclose=False) as store: - with open_dataset(store) as ds: - expected = Dataset({'x': ((), 42)}) - assert_identical(expected, ds) + store = backends.NetCDF4DataStore(nc) + with open_dataset(store) as ds: + expected = Dataset({'x': ((), 42)}) + assert_identical(expected, ds) def test_read_variable_len_strings(self): with create_tmp_file() as tmp_file: @@ -1181,7 +1178,6 @@ def test_read_variable_len_strings(self): @requires_netCDF4 class NetCDF4DataTest(BaseNetCDF4Test): - autoclose = False @contextlib.contextmanager def create_store(self): @@ -1247,9 +1243,13 @@ def test_setncattr_string(self): totest.attrs['bar']) assert one_string == totest.attrs['baz'] - -class NetCDF4DataStoreAutocloseTrue(NetCDF4DataTest): - autoclose = True + def test_autoclose_future_warning(self): + data = create_test_data() + with create_tmp_file() as tmp_file: + self.save(data, tmp_file) + with pytest.warns(FutureWarning): + with self.open(tmp_file, autoclose=True) as actual: + assert_identical(data, actual) @requires_netCDF4 @@ -1290,10 +1290,6 @@ def test_write_inconsistent_chunks(self): assert actual['y'].encoding['chunksizes'] == (100, 50) -class NetCDF4ViaDaskDataTestAutocloseTrue(NetCDF4ViaDaskDataTest): - autoclose = True - - @requires_zarr class BaseZarrTest(CFEncodedDataTest): @@ -1571,19 +1567,14 @@ def test_to_netcdf_explicit_engine(self): # regression test for GH1321 Dataset({'foo': 42}).to_netcdf(engine='scipy') - @pytest.mark.skipif(PY2, reason='cannot pickle BytesIO on Python 2') - def test_bytesio_pickle(self): + def test_bytes_pickle(self): data = Dataset({'foo': ('x', [1, 2, 3])}) - fobj = BytesIO(data.to_netcdf()) - with open_dataset(fobj, autoclose=self.autoclose) as ds: + fobj = data.to_netcdf() + with self.open(fobj) as ds: unpickled = pickle.loads(pickle.dumps(ds)) assert_identical(unpickled, data) -class ScipyInMemoryDataTestAutocloseTrue(ScipyInMemoryDataTest): - autoclose = True - - @requires_scipy class ScipyFileObjectTest(ScipyWriteTest): engine = 'scipy' @@ -1649,10 +1640,6 @@ def test_nc4_scipy(self): open_dataset(tmp_file, engine='scipy') -class ScipyFilePathTestAutocloseTrue(ScipyFilePathTest): - autoclose = True - - @requires_netCDF4 class NetCDF3ViaNetCDF4DataTest(CFEncodedDataTest, NetCDF3Only): engine = 'netcdf4' @@ -1673,10 +1660,6 @@ def test_encoding_kwarg_vlen_string(self): pass -class NetCDF3ViaNetCDF4DataTestAutocloseTrue(NetCDF3ViaNetCDF4DataTest): - autoclose = True - - @requires_netCDF4 class NetCDF4ClassicViaNetCDF4DataTest(CFEncodedDataTest, NetCDF3Only, object): @@ -1691,11 +1674,6 @@ def create_store(self): yield store -class NetCDF4ClassicViaNetCDF4DataTestAutocloseTrue( - NetCDF4ClassicViaNetCDF4DataTest): - autoclose = True - - @requires_scipy_or_netCDF4 class GenericNetCDFDataTest(CFEncodedDataTest, NetCDF3Only): # verify that we can read and write netCDF3 files as long as we have scipy @@ -1772,10 +1750,6 @@ def test_encoding_unlimited_dims(self): assert_equal(ds, actual) -class GenericNetCDFDataTestAutocloseTrue(GenericNetCDFDataTest): - autoclose = True - - @requires_h5netcdf @requires_netCDF4 class H5NetCDFDataTest(BaseNetCDF4Test): @@ -1789,8 +1763,11 @@ def create_store(self): @pytest.mark.filterwarnings('ignore:complex dtypes are supported by h5py') def test_complex(self): expected = Dataset({'x': ('y', np.ones(5) + 1j * np.ones(5))}) - with self.roundtrip(expected) as actual: - assert_equal(expected, actual) + with pytest.warns(FutureWarning): + # TODO: make it possible to write invalid netCDF files from xarray + # without a warning + with self.roundtrip(expected) as actual: + assert_equal(expected, actual) @pytest.mark.xfail(reason='https://github.com/pydata/xarray/issues/535') def test_cross_engine_read_write_netcdf4(self): @@ -1905,25 +1882,24 @@ def test_dump_encodings_h5py(self): assert actual.x.encoding['compression_opts'] is None -# tests pending h5netcdf fix -@unittest.skip -class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest): - autoclose = True - - @pytest.fixture(params=['scipy', 'netcdf4', 'h5netcdf', 'pynio']) def readengine(request): return request.param -@pytest.fixture(params=[1, 100]) +@pytest.fixture(params=[1, 20]) def nfiles(request): return request.param -@pytest.fixture(params=[True, False]) -def autoclose(request): - return request.param +@pytest.fixture(params=[5, None]) +def file_cache_maxsize(request): + maxsize = request.param + if maxsize is not None: + with set_options(file_cache_maxsize=maxsize): + yield maxsize + else: + yield maxsize @pytest.fixture(params=[True, False]) @@ -1946,8 +1922,8 @@ def skip_if_not_engine(engine): pytest.importorskip(engine) -def test_open_mfdataset_manyfiles(readengine, nfiles, autoclose, parallel, - chunks): +def test_open_mfdataset_manyfiles(readengine, nfiles, parallel, chunks, + file_cache_maxsize): # skip certain combinations skip_if_not_engine(readengine) @@ -1955,9 +1931,6 @@ def test_open_mfdataset_manyfiles(readengine, nfiles, autoclose, parallel, if not has_dask and parallel: pytest.skip('parallel requires dask') - if readengine == 'h5netcdf' and autoclose: - pytest.skip('h5netcdf does not support autoclose yet') - if ON_WINDOWS: pytest.skip('Skipping on Windows') @@ -1973,7 +1946,7 @@ def test_open_mfdataset_manyfiles(readengine, nfiles, autoclose, parallel, # check that calculation on opened datasets works properly actual = open_mfdataset(tmpfiles, engine=readengine, parallel=parallel, - autoclose=autoclose, chunks=chunks) + chunks=chunks) # check that using open_mfdataset returns dask arrays for variables assert isinstance(actual['foo'].data, dask_array_type) @@ -2172,22 +2145,20 @@ def test_open_mfdataset(self): with create_tmp_file() as tmp2: original.isel(x=slice(5)).to_netcdf(tmp1) original.isel(x=slice(5, 10)).to_netcdf(tmp2) - with open_mfdataset([tmp1, tmp2], - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp1, tmp2]) as actual: assert isinstance(actual.foo.variable.data, da.Array) assert actual.foo.variable.data.chunks == \ ((5, 5),) assert_identical(original, actual) - with open_mfdataset([tmp1, tmp2], chunks={'x': 3}, - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp1, tmp2], chunks={'x': 3}) as actual: assert actual.foo.variable.data.chunks == \ ((3, 2, 3, 2),) with raises_regex(IOError, 'no files to open'): - open_mfdataset('foo-bar-baz-*.nc', autoclose=self.autoclose) + open_mfdataset('foo-bar-baz-*.nc') with raises_regex(ValueError, 'wild-card'): - open_mfdataset('http://some/remote/uri', autoclose=self.autoclose) + open_mfdataset('http://some/remote/uri') @requires_pathlib def test_open_mfdataset_pathlib(self): @@ -2198,8 +2169,7 @@ def test_open_mfdataset_pathlib(self): tmp2 = Path(tmp2) original.isel(x=slice(5)).to_netcdf(tmp1) original.isel(x=slice(5, 10)).to_netcdf(tmp2) - with open_mfdataset([tmp1, tmp2], - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp1, tmp2]) as actual: assert_identical(original, actual) def test_attrs_mfdataset(self): @@ -2230,8 +2200,7 @@ def preprocess(ds): return ds.assign_coords(z=0) expected = preprocess(original) - with open_mfdataset(tmp, preprocess=preprocess, - autoclose=self.autoclose) as actual: + with open_mfdataset(tmp, preprocess=preprocess) as actual: assert_identical(expected, actual) def test_save_mfdataset_roundtrip(self): @@ -2241,8 +2210,7 @@ def test_save_mfdataset_roundtrip(self): with create_tmp_file() as tmp1: with create_tmp_file() as tmp2: save_mfdataset(datasets, [tmp1, tmp2]) - with open_mfdataset([tmp1, tmp2], - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp1, tmp2]) as actual: assert_identical(actual, original) def test_save_mfdataset_invalid(self): @@ -2268,15 +2236,14 @@ def test_save_mfdataset_pathlib_roundtrip(self): tmp1 = Path(tmp1) tmp2 = Path(tmp2) save_mfdataset(datasets, [tmp1, tmp2]) - with open_mfdataset([tmp1, tmp2], - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp1, tmp2]) as actual: assert_identical(actual, original) def test_open_and_do_math(self): original = Dataset({'foo': ('x', np.random.randn(10))}) with create_tmp_file() as tmp: original.to_netcdf(tmp) - with open_mfdataset(tmp, autoclose=self.autoclose) as ds: + with open_mfdataset(tmp) as ds: actual = 1.0 * ds assert_allclose(original, actual, decode_bytes=False) @@ -2286,8 +2253,7 @@ def test_open_mfdataset_concat_dim_none(self): data = Dataset({'x': 0}) data.to_netcdf(tmp1) Dataset({'x': np.nan}).to_netcdf(tmp2) - with open_mfdataset([tmp1, tmp2], concat_dim=None, - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp1, tmp2], concat_dim=None) as actual: assert_identical(data, actual) def test_open_dataset(self): @@ -2314,8 +2280,7 @@ def test_open_single_dataset(self): {'baz': [100]}) with create_tmp_file() as tmp: original.to_netcdf(tmp) - with open_mfdataset([tmp], concat_dim=dim, - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp], concat_dim=dim) as actual: assert_identical(expected, actual) def test_dask_roundtrip(self): @@ -2334,10 +2299,10 @@ def test_deterministic_names(self): with create_tmp_file() as tmp: data = create_test_data() data.to_netcdf(tmp) - with open_mfdataset(tmp, autoclose=self.autoclose) as ds: + with open_mfdataset(tmp) as ds: original_names = dict((k, v.data.name) for k, v in ds.data_vars.items()) - with open_mfdataset(tmp, autoclose=self.autoclose) as ds: + with open_mfdataset(tmp) as ds: repeat_names = dict((k, v.data.name) for k, v in ds.data_vars.items()) for var_name, dask_name in original_names.items(): @@ -2355,41 +2320,22 @@ def test_dataarray_compute(self): assert computed._in_memory assert_allclose(actual, computed, decode_bytes=False) - def test_to_netcdf_compute_false_roundtrip(self): - from dask.delayed import Delayed - - original = create_test_data().chunk() - - with create_tmp_file() as tmp_file: - # dataset, path, **kwargs): - delayed_obj = self.save(original, tmp_file, compute=False) - assert isinstance(delayed_obj, Delayed) - delayed_obj.compute() - - with self.open(tmp_file) as actual: - assert_identical(original, actual) - def test_save_mfdataset_compute_false_roundtrip(self): from dask.delayed import Delayed original = Dataset({'foo': ('x', np.random.randn(10))}).chunk() datasets = [original.isel(x=slice(5)), original.isel(x=slice(5, 10))] - with create_tmp_file() as tmp1: - with create_tmp_file() as tmp2: + with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as tmp1: + with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as tmp2: delayed_obj = save_mfdataset(datasets, [tmp1, tmp2], engine=self.engine, compute=False) assert isinstance(delayed_obj, Delayed) delayed_obj.compute() - with open_mfdataset([tmp1, tmp2], - autoclose=self.autoclose) as actual: + with open_mfdataset([tmp1, tmp2]) as actual: assert_identical(actual, original) -class DaskTestAutocloseTrue(DaskTest): - autoclose = True - - @requires_scipy_or_netCDF4 @requires_pydap class PydapTest(object): @@ -2500,8 +2446,7 @@ def test_write_store(self): @contextlib.contextmanager def open(self, path, **kwargs): - with open_dataset(path, engine='pynio', autoclose=self.autoclose, - **kwargs) as ds: + with open_dataset(path, engine='pynio', **kwargs) as ds: yield ds def save(self, dataset, path, **kwargs): @@ -2519,19 +2464,12 @@ def test_weakrefs(self): assert_identical(actual, expected) -class PyNioTestAutocloseTrue(PyNioTest): - autoclose = True - - @requires_pseudonetcdf @pytest.mark.filterwarnings('ignore:IOAPI_ISPH is assumed to be 6370000') class PseudoNetCDFFormatTest(object): - autoclose = True def open(self, path, **kwargs): - return open_dataset(path, engine='pseudonetcdf', - autoclose=self.autoclose, - **kwargs) + return open_dataset(path, engine='pseudonetcdf', **kwargs) @contextlib.contextmanager def roundtrip(self, data, save_kwargs={}, open_kwargs={}, @@ -2548,7 +2486,6 @@ def test_ict_format(self): """ ictfile = open_example_dataset('example.ict', engine='pseudonetcdf', - autoclose=False, backend_kwargs={'format': 'ffi1001'}) stdattr = { 'fill_value': -9999.0, @@ -2646,7 +2583,6 @@ def test_ict_format_write(self): fmtkw = {'format': 'ffi1001'} expected = open_example_dataset('example.ict', engine='pseudonetcdf', - autoclose=False, backend_kwargs=fmtkw) with self.roundtrip(expected, save_kwargs=fmtkw, open_kwargs={'backend_kwargs': fmtkw}) as actual: @@ -2659,7 +2595,6 @@ def test_uamiv_format_read(self): camxfile = open_example_dataset('example.uamiv', engine='pseudonetcdf', - autoclose=True, backend_kwargs={'format': 'uamiv'}) data = np.arange(20, dtype='f').reshape(1, 1, 4, 5) expected = xr.Variable(('TSTEP', 'LAY', 'ROW', 'COL'), data, @@ -2687,7 +2622,6 @@ def test_uamiv_format_mfread(self): ['example.uamiv', 'example.uamiv'], engine='pseudonetcdf', - autoclose=True, concat_dim='TSTEP', backend_kwargs={'format': 'uamiv'}) @@ -2701,11 +2635,11 @@ def test_uamiv_format_mfread(self): data1 = np.array(['2002-06-03'], 'datetime64[ns]') data = np.concatenate([data1] * 2, axis=0) - expected = xr.Variable(('TSTEP',), data, - dict(bounds='time_bounds', - long_name=('synthesized time coordinate ' + - 'from SDATE, STIME, STEP ' + - 'global attributes'))) + attrs = dict(bounds='time_bounds', + long_name=('synthesized time coordinate ' + + 'from SDATE, STIME, STEP ' + + 'global attributes')) + expected = xr.Variable(('TSTEP',), data, attrs) actual = camxfile.variables['time'] assert_allclose(expected, actual) camxfile.close() @@ -2715,7 +2649,6 @@ def test_uamiv_format_write(self): expected = open_example_dataset('example.uamiv', engine='pseudonetcdf', - autoclose=False, backend_kwargs=fmtkw) with self.roundtrip(expected, save_kwargs=fmtkw, @@ -3312,32 +3245,6 @@ def test_dataarray_to_netcdf_no_name_pathlib(self): assert_identical(original_da, loaded_da) -def test_pickle_reconstructor(): - - lines = ['foo bar spam eggs'] - - with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as tmp: - with open(tmp, 'w') as f: - f.writelines(lines) - - obj = PickleByReconstructionWrapper(open, tmp) - - assert obj.value.readlines() == lines - - p_obj = pickle.dumps(obj) - obj.value.close() # for windows - obj2 = pickle.loads(p_obj) - - assert obj2.value.readlines() == lines - - # roundtrip again to make sure we can fully restore the state - p_obj2 = pickle.dumps(obj2) - obj2.value.close() # for windows - obj3 = pickle.loads(p_obj2) - - assert obj3.value.readlines() == lines - - @requires_scipy_or_netCDF4 def test_no_warning_from_dask_effective_get(): with create_tmp_file() as tmpfile: diff --git a/xarray/tests/test_backends_file_manager.py b/xarray/tests/test_backends_file_manager.py new file mode 100644 index 00000000000..591c981cd45 --- /dev/null +++ b/xarray/tests/test_backends_file_manager.py @@ -0,0 +1,114 @@ +import pickle +import threading +try: + from unittest import mock +except ImportError: + import mock # noqa: F401 + +import pytest + +from xarray.backends.file_manager import CachingFileManager +from xarray.backends.lru_cache import LRUCache + + +@pytest.fixture(params=[1, 2, 3, None]) +def file_cache(request): + maxsize = request.param + if maxsize is None: + yield {} + else: + yield LRUCache(maxsize) + + +def test_file_manager_mock_write(file_cache): + mock_file = mock.Mock() + opener = mock.Mock(spec=open, return_value=mock_file) + lock = mock.MagicMock(spec=threading.Lock()) + + manager = CachingFileManager( + opener, 'filename', lock=lock, cache=file_cache) + f = manager.acquire() + f.write('contents') + manager.close() + + assert not file_cache + opener.assert_called_once_with('filename') + mock_file.write.assert_called_once_with('contents') + mock_file.close.assert_called_once_with() + lock.__enter__.assert_has_calls([mock.call(), mock.call()]) + + +def test_file_manager_write_consecutive(tmpdir, file_cache): + path1 = str(tmpdir.join('testing1.txt')) + path2 = str(tmpdir.join('testing2.txt')) + manager1 = CachingFileManager(open, path1, mode='w', cache=file_cache) + manager2 = CachingFileManager(open, path2, mode='w', cache=file_cache) + f1a = manager1.acquire() + f1a.write('foo') + f1a.flush() + f2 = manager2.acquire() + f2.write('bar') + f2.flush() + f1b = manager1.acquire() + f1b.write('baz') + assert (getattr(file_cache, 'maxsize', float('inf')) > 1) == (f1a is f1b) + manager1.close() + manager2.close() + + with open(path1, 'r') as f: + assert f.read() == 'foobaz' + with open(path2, 'r') as f: + assert f.read() == 'bar' + + +def test_file_manager_write_concurrent(tmpdir, file_cache): + path = str(tmpdir.join('testing.txt')) + manager = CachingFileManager(open, path, mode='w', cache=file_cache) + f1 = manager.acquire() + f2 = manager.acquire() + f3 = manager.acquire() + assert f1 is f2 + assert f2 is f3 + f1.write('foo') + f1.flush() + f2.write('bar') + f2.flush() + f3.write('baz') + f3.flush() + manager.close() + + with open(path, 'r') as f: + assert f.read() == 'foobarbaz' + + +def test_file_manager_write_pickle(tmpdir, file_cache): + path = str(tmpdir.join('testing.txt')) + manager = CachingFileManager(open, path, mode='w', cache=file_cache) + f = manager.acquire() + f.write('foo') + f.flush() + manager2 = pickle.loads(pickle.dumps(manager)) + f2 = manager2.acquire() + f2.write('bar') + manager2.close() + manager.close() + + with open(path, 'r') as f: + assert f.read() == 'foobar' + + +def test_file_manager_read(tmpdir, file_cache): + path = str(tmpdir.join('testing.txt')) + + with open(path, 'w') as f: + f.write('foobar') + + manager = CachingFileManager(open, path, cache=file_cache) + f = manager.acquire() + assert f.read() == 'foobar' + manager.close() + + +def test_file_manager_invalid_kwargs(): + with pytest.raises(TypeError): + CachingFileManager(open, 'dummy', mode='w', invalid=True) diff --git a/xarray/tests/test_backends_locks.py b/xarray/tests/test_backends_locks.py new file mode 100644 index 00000000000..5f83321802e --- /dev/null +++ b/xarray/tests/test_backends_locks.py @@ -0,0 +1,13 @@ +import threading + +from xarray.backends import locks + + +def test_threaded_lock(): + lock1 = locks._get_threaded_lock('foo') + assert isinstance(lock1, type(threading.Lock())) + lock2 = locks._get_threaded_lock('foo') + assert lock1 is lock2 + + lock3 = locks._get_threaded_lock('bar') + assert lock1 is not lock3 diff --git a/xarray/tests/test_backends_lru_cache.py b/xarray/tests/test_backends_lru_cache.py new file mode 100644 index 00000000000..03eb6dcf208 --- /dev/null +++ b/xarray/tests/test_backends_lru_cache.py @@ -0,0 +1,91 @@ +try: + from unittest import mock +except ImportError: + import mock # noqa: F401 + +import pytest + +from xarray.backends.lru_cache import LRUCache + + +def test_simple(): + cache = LRUCache(maxsize=2) + cache['x'] = 1 + cache['y'] = 2 + + assert cache['x'] == 1 + assert cache['y'] == 2 + assert len(cache) == 2 + assert dict(cache) == {'x': 1, 'y': 2} + assert list(cache.keys()) == ['x', 'y'] + assert list(cache.items()) == [('x', 1), ('y', 2)] + + cache['z'] = 3 + assert len(cache) == 2 + assert list(cache.items()) == [('y', 2), ('z', 3)] + + +def test_trivial(): + cache = LRUCache(maxsize=0) + cache['x'] = 1 + assert len(cache) == 0 + + +def test_invalid(): + with pytest.raises(TypeError): + LRUCache(maxsize=None) + with pytest.raises(ValueError): + LRUCache(maxsize=-1) + + +def test_update_priority(): + cache = LRUCache(maxsize=2) + cache['x'] = 1 + cache['y'] = 2 + assert list(cache) == ['x', 'y'] + assert 'x' in cache # contains + assert list(cache) == ['y', 'x'] + assert cache['y'] == 2 # getitem + assert list(cache) == ['x', 'y'] + cache['x'] = 3 # setitem + assert list(cache.items()) == [('y', 2), ('x', 3)] + + +def test_del(): + cache = LRUCache(maxsize=2) + cache['x'] = 1 + cache['y'] = 2 + del cache['x'] + assert dict(cache) == {'y': 2} + + +def test_on_evict(): + on_evict = mock.Mock() + cache = LRUCache(maxsize=1, on_evict=on_evict) + cache['x'] = 1 + cache['y'] = 2 + on_evict.assert_called_once_with('x', 1) + + +def test_on_evict_trivial(): + on_evict = mock.Mock() + cache = LRUCache(maxsize=0, on_evict=on_evict) + cache['x'] = 1 + on_evict.assert_called_once_with('x', 1) + + +def test_resize(): + cache = LRUCache(maxsize=2) + assert cache.maxsize == 2 + cache['w'] = 0 + cache['x'] = 1 + cache['y'] = 2 + assert list(cache.items()) == [('x', 1), ('y', 2)] + cache.maxsize = 10 + cache['z'] = 3 + assert list(cache.items()) == [('x', 1), ('y', 2), ('z', 3)] + cache.maxsize = 1 + assert list(cache.items()) == [('z', 3)] + + with pytest.raises(ValueError): + cache.maxsize = -1 diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 9bee965392b..89704653e92 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -64,8 +64,8 @@ def create_test_multiindex(): class InaccessibleVariableDataStore(backends.InMemoryDataStore): - def __init__(self, writer=None): - super(InaccessibleVariableDataStore, self).__init__(writer) + def __init__(self): + super(InaccessibleVariableDataStore, self).__init__() self._indexvars = set() def store(self, variables, *args, **kwargs): diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 32035afdc57..7c77a62d3c9 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -15,12 +15,13 @@ from distributed.utils_test import cluster, gen_cluster from distributed.utils_test import loop # flake8: noqa from distributed.client import futures_of +import numpy as np import xarray as xr +from xarray.backends.locks import HDF5_LOCK, CombinedLock from xarray.tests.test_backends import (ON_WINDOWS, create_tmp_file, create_tmp_geotiff) from xarray.tests.test_dataset import create_test_data -from xarray.backends.common import HDF5_LOCK, CombinedLock from . import ( assert_allclose, has_h5netcdf, has_netCDF4, requires_rasterio, has_scipy, @@ -33,6 +34,11 @@ da = pytest.importorskip('dask.array') +@pytest.fixture +def tmp_netcdf_filename(tmpdir): + return str(tmpdir.join('testfile.nc')) + + ENGINES = [] if has_scipy: ENGINES.append('scipy') @@ -45,81 +51,69 @@ 'NETCDF3_64BIT_DATA', 'NETCDF4_CLASSIC', 'NETCDF4'], 'scipy': ['NETCDF3_CLASSIC', 'NETCDF3_64BIT'], 'h5netcdf': ['NETCDF4']} -TEST_FORMATS = ['NETCDF3_CLASSIC', 'NETCDF4_CLASSIC', 'NETCDF4'] +ENGINES_AND_FORMATS = [ + ('netcdf4', 'NETCDF3_CLASSIC'), + ('netcdf4', 'NETCDF4_CLASSIC'), + ('netcdf4', 'NETCDF4'), + ('h5netcdf', 'NETCDF4'), + ('scipy', 'NETCDF3_64BIT'), +] -@pytest.mark.xfail(sys.platform == 'win32', - reason='https://github.com/pydata/xarray/issues/1738') -@pytest.mark.parametrize('engine', ['netcdf4']) -@pytest.mark.parametrize('autoclose', [True, False]) -@pytest.mark.parametrize('nc_format', TEST_FORMATS) -def test_dask_distributed_netcdf_roundtrip(monkeypatch, loop, - engine, autoclose, nc_format): - monkeypatch.setenv('HDF5_USE_FILE_LOCKING', 'FALSE') - - chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} - - with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: - with cluster() as (s, [a, b]): - with Client(s['address'], loop=loop) as c: +@pytest.mark.parametrize('engine,nc_format', ENGINES_AND_FORMATS) +def test_dask_distributed_netcdf_roundtrip( + loop, tmp_netcdf_filename, engine, nc_format): - original = create_test_data().chunk(chunks) - original.to_netcdf(filename, engine=engine, format=nc_format) - - with xr.open_dataset(filename, - chunks=chunks, - engine=engine, - autoclose=autoclose) as restored: - assert isinstance(restored.var1.data, da.Array) - computed = restored.compute() - assert_allclose(original, computed) + if engine not in ENGINES: + pytest.skip('engine not available') + chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} -@pytest.mark.xfail(sys.platform == 'win32', - reason='https://github.com/pydata/xarray/issues/1738') -@pytest.mark.parametrize('engine', ENGINES) -@pytest.mark.parametrize('autoclose', [True, False]) -@pytest.mark.parametrize('nc_format', TEST_FORMATS) -def test_dask_distributed_read_netcdf_integration_test(loop, engine, autoclose, - nc_format): + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: - if engine == 'h5netcdf' and autoclose: - pytest.skip('h5netcdf does not support autoclose') + original = create_test_data().chunk(chunks) - if nc_format not in NC_FORMATS[engine]: - pytest.skip('invalid format for engine') + if engine == 'scipy': + with pytest.raises(NotImplementedError): + original.to_netcdf(tmp_netcdf_filename, + engine=engine, format=nc_format) + return - chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} + original.to_netcdf(tmp_netcdf_filename, + engine=engine, format=nc_format) - with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: - with cluster() as (s, [a, b]): - with Client(s['address'], loop=loop) as c: + with xr.open_dataset(tmp_netcdf_filename, + chunks=chunks, engine=engine) as restored: + assert isinstance(restored.var1.data, da.Array) + computed = restored.compute() + assert_allclose(original, computed) - original = create_test_data() - original.to_netcdf(filename, engine=engine, format=nc_format) - with xr.open_dataset(filename, - chunks=chunks, - engine=engine, - autoclose=autoclose) as restored: - assert isinstance(restored.var1.data, da.Array) - computed = restored.compute() - assert_allclose(original, computed) +@pytest.mark.parametrize('engine,nc_format', ENGINES_AND_FORMATS) +def test_dask_distributed_read_netcdf_integration_test( + loop, tmp_netcdf_filename, engine, nc_format): + if engine not in ENGINES: + pytest.skip('engine not available') -@pytest.mark.parametrize('engine', ['h5netcdf', 'scipy']) -def test_dask_distributed_netcdf_integration_test_not_implemented(loop, engine): chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} - with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: - with cluster() as (s, [a, b]): - with Client(s['address'], loop=loop) as c: + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: + + original = create_test_data() + original.to_netcdf(tmp_netcdf_filename, + engine=engine, format=nc_format) - original = create_test_data().chunk(chunks) + with xr.open_dataset(tmp_netcdf_filename, + chunks=chunks, + engine=engine) as restored: + assert isinstance(restored.var1.data, da.Array) + computed = restored.compute() + assert_allclose(original, computed) - with raises_regex(NotImplementedError, 'distributed'): - original.to_netcdf(filename, engine=engine) @requires_zarr diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index aed96f1acb6..4441375a1b1 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -4,6 +4,7 @@ import xarray from xarray.core.options import OPTIONS +from xarray.backends.file_manager import FILE_CACHE def test_invalid_option_raises(): @@ -11,6 +12,38 @@ def test_invalid_option_raises(): xarray.set_options(not_a_valid_options=True) +def test_display_width(): + with pytest.raises(ValueError): + xarray.set_options(display_width=0) + with pytest.raises(ValueError): + xarray.set_options(display_width=-10) + with pytest.raises(ValueError): + xarray.set_options(display_width=3.5) + + +def test_arithmetic_join(): + with pytest.raises(ValueError): + xarray.set_options(arithmetic_join='invalid') + with xarray.set_options(arithmetic_join='exact'): + assert OPTIONS['arithmetic_join'] == 'exact' + + +def test_enable_cftimeindex(): + with pytest.raises(ValueError): + xarray.set_options(enable_cftimeindex=None) + with xarray.set_options(enable_cftimeindex=True): + assert OPTIONS['enable_cftimeindex'] + + +def test_file_cache_maxsize(): + with pytest.raises(ValueError): + xarray.set_options(file_cache_maxsize=0) + original_size = FILE_CACHE.maxsize + with xarray.set_options(file_cache_maxsize=123): + assert FILE_CACHE.maxsize == 123 + assert FILE_CACHE.maxsize == original_size + + def test_nested_options(): original = OPTIONS['display_width'] with xarray.set_options(display_width=1): From 6840fc2c0e38d58ec788c988bffb1ce399f8827b Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 9 Oct 2018 22:26:03 -0700 Subject: [PATCH 245/282] Ignore W504 with pep8speaks (#2474) --- .pep8speaks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pep8speaks.yml b/.pep8speaks.yml index cd610907007..aedce6e44eb 100644 --- a/.pep8speaks.yml +++ b/.pep8speaks.yml @@ -9,3 +9,4 @@ pycodestyle: - E402, # module level import not at top of file - E731, # do not assign a lambda expression, use a def - W503 # line break before binary operator + - W504 # line break after binary operator From 7f20a20aa278d2bb056403d665c10e29968755cd Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Wed, 10 Oct 2018 15:47:22 +0200 Subject: [PATCH 246/282] Inhouse LooseVersion (#2477) * Inhouse LooseVersion * Made LooseVersion a function --- xarray/tests/__init__.py | 9 ++++++++- xarray/tests/test_dataarray.py | 14 ++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 285c1f03a26..5f724dd6713 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -3,7 +3,7 @@ from __future__ import print_function import warnings from contextlib import contextmanager -from distutils.version import LooseVersion +from distutils import version import re import importlib @@ -53,6 +53,13 @@ def _importorskip(modname, minversion=None): return has, func +def LooseVersion(vstring): + # Our development version is something like '0.10.9+aac7bfc' + # This function just ignored the git commit id. + vstring = vstring.split('+')[0] + return version.LooseVersion(vstring) + + has_matplotlib, requires_matplotlib = _importorskip('matplotlib') has_matplotlib2, requires_matplotlib2 = _importorskip('matplotlib', minversion='2') diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index d15a0bb6081..e49b6cdf517 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -17,10 +17,10 @@ from xarray.core.common import ALL_DIMS, full_like from xarray.core.pycompat import OrderedDict, iteritems from xarray.tests import ( - ReturnItem, assert_allclose, assert_array_equal, assert_equal, - assert_identical, raises_regex, requires_bottleneck, requires_cftime, - requires_dask, requires_iris, requires_np113, requires_scipy, - source_ndarray) + LooseVersion, ReturnItem, assert_allclose, assert_array_equal, + assert_equal, assert_identical, raises_regex, requires_bottleneck, + requires_cftime, requires_dask, requires_iris, requires_np113, + requires_scipy, source_ndarray) class TestDataArray(object): @@ -2026,10 +2026,8 @@ def test_groupby_warning(self): with pytest.warns(FutureWarning): grouped.sum() - # Currently disabled due to https://github.com/pydata/xarray/issues/2468 - # @pytest.mark.skipif(LooseVersion(xr.__version__) < LooseVersion('0.12'), - # reason="not to forget the behavior change") - @pytest.mark.skip + @pytest.mark.skipif(LooseVersion(xr.__version__) < LooseVersion('0.12'), + reason="not to forget the behavior change") def test_groupby_sum_default(self): array = self.make_groupby_example_array() grouped = array.groupby('abc') From 81172ecaf0c8ffec9e688fe6482680475f2e12c2 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 11 Oct 2018 19:28:24 +0900 Subject: [PATCH 247/282] Line plots with 2D coordinates (#2408) * 2D co-ordinate plotting. * Align & transpose hue variable too * bugfix. * Remove transpose kwarg and ensure hue is dimension instead * start adding docs. * Test + nicer error msg. * Add whats-new. * Address review comments. * Remove old error message about hue with 2d data. * flake8 isort linting * more fixes. * Fix whats-new. * pep8 * fix bad merge --- doc/plotting.rst | 9 +++++++ doc/whats-new.rst | 4 +++ xarray/plot/plot.py | 54 ++++++++++++++++++++++++++++----------- xarray/plot/utils.py | 4 +-- xarray/tests/test_plot.py | 19 ++++++++++++++ 5 files changed, 73 insertions(+), 17 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index 43faa83b9da..37fdf03804a 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -702,3 +702,12 @@ You can however decide to infer the cell boundaries and use the outside the xarray framework. .. _cell boundaries: http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#cell-boundaries + +One can also make line plots with multidimensional coordinates. In this case, ``hue`` must be a dimension name, not a coordinate name. + +.. ipython:: python + + f, ax = plt.subplots(2, 1) + da.plot.line(x='lon', hue='y', ax=ax[0]); + @savefig plotting_example_2d_hue_xy.png + da.plot.line(x='lon', hue='x', ax=ax[1]); diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d0fec7b0778..c8ae5ac43c8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -57,6 +57,10 @@ Documentation Enhancements ~~~~~~~~~~~~ +- :py:meth:`xarray.DataArray.plot.line` can now accept multidimensional + coordinate variables as input. `hue` must be a dimension name in this case. + (:issue:`2407`) + By `Deepak Cherian `_. - Added support for Python 3.7. (:issue:`2271`). By `Joe Hamman `_. - Added :py:meth:`~xarray.CFTimeIndex.shift` for shifting the values of a diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index b44ae7b3856..a43cee14eb3 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -14,6 +14,7 @@ import numpy as np import pandas as pd +from xarray.core.alignment import align from xarray.core.common import contains_cftime_datetimes from xarray.core.pycompat import basestring @@ -173,8 +174,10 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, hue=None, kwargs['hue'] = hue elif ndims == 2: if hue: - raise ValueError('hue is not compatible with 2d data') - plotfunc = pcolormesh + plotfunc = line + kwargs['hue'] = hue + else: + plotfunc = pcolormesh else: if row or col or hue: raise ValueError(error_msg) @@ -190,10 +193,10 @@ def _infer_line_data(darray, x, y, hue): .format(', '.join([repr(dd) for dd in darray.dims]))) ndims = len(darray.dims) - if x is not None and x not in darray.dims: + if x is not None and x not in darray.dims and x not in darray.coords: raise ValueError('x ' + error_msg) - if y is not None and y not in darray.dims: + if y is not None and y not in darray.dims and y not in darray.coords: raise ValueError('y ' + error_msg) if x is not None and y is not None: @@ -207,11 +210,11 @@ def _infer_line_data(darray, x, y, hue): huelabel = '' if (x is None and y is None) or x == dim: - xplt = darray.coords[dim] + xplt = darray[dim] yplt = darray else: - yplt = darray.coords[dim] + yplt = darray[dim] xplt = darray else: @@ -221,18 +224,37 @@ def _infer_line_data(darray, x, y, hue): if y is None: xname, huename = _infer_xy_labels(darray=darray, x=x, y=hue) - yname = darray.name - xplt = darray.coords[xname] - yplt = darray.transpose(xname, huename) + xplt = darray[xname] + if xplt.ndim > 1: + if huename in darray.dims: + otherindex = 1 if darray.dims.index(huename) == 0 else 0 + otherdim = darray.dims[otherindex] + yplt = darray.transpose(otherdim, huename) + xplt = xplt.transpose(otherdim, huename) + else: + raise ValueError('For 2D inputs, hue must be a dimension' + + ' i.e. one of ' + repr(darray.dims)) + + else: + yplt = darray.transpose(xname, huename) else: yname, huename = _infer_xy_labels(darray=darray, x=y, y=hue) - xname = darray.name - xplt = darray.transpose(yname, huename) - yplt = darray.coords[yname] + yplt = darray[yname] + if yplt.ndim > 1: + if huename in darray.dims: + otherindex = 1 if darray.dims.index(huename) == 0 else 0 + xplt = darray.transpose(otherdim, huename) + else: + raise ValueError('For 2D inputs, hue must be a dimension' + + ' i.e. one of ' + repr(darray.dims)) + + else: + xplt = darray.transpose(yname, huename) - hueplt = darray.coords[huename] huelabel = label_from_attrs(darray[huename]) + hueplt = darray[huename] + xlabel = label_from_attrs(xplt) ylabel = label_from_attrs(yplt) @@ -265,9 +287,11 @@ def line(darray, *args, **kwargs): Axis on which to plot this figure. By default, use the current axis. Mutually exclusive with ``size`` and ``figsize``. hue : string, optional - Coordinate for which you want multiple lines plotted. + Dimension or coordinate for which you want multiple lines plotted. + If plotting against a 2D coordinate, ``hue`` must be a dimension. x, y : string, optional - Coordinates for x, y axis. Only one of these may be specified. + Dimensions or coordinates for x, y axis. + Only one of these may be specified. The other coordinate plots values from the DataArray on which this plot method is called. xscale, yscale : 'linear', 'symlog', 'log', 'logit', optional diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index be38a6d7a4c..111fd4de6e5 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -326,11 +326,11 @@ def _infer_xy_labels(darray, x, y, imshow=False, rgb=None): raise ValueError('DataArray must be 2d') y, x = darray.dims elif x is None: - if y not in darray.dims: + if y not in darray.dims and y not in darray.coords: raise ValueError('y must be a dimension name if x is not supplied') x = darray.dims[0] if y == darray.dims[1] else darray.dims[1] elif y is None: - if x not in darray.dims: + if x not in darray.dims and x not in darray.coords: raise ValueError('x must be a dimension name if y is not supplied') y = darray.dims[0] if x == darray.dims[1] else darray.dims[1] elif any(k not in darray.coords and k not in darray.dims for k in (x, y)): diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 53f6077ee4f..d2a43bc5b52 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -181,6 +181,25 @@ def test_2d_line_accepts_hue_kw(self): assert (plt.gca().get_legend().get_title().get_text() == 'dim_1') + def test_2d_coords_line_plot(self): + lon, lat = np.meshgrid(np.linspace(-20, 20, 5), + np.linspace(0, 30, 4)) + lon += lat / 10 + lat += lon / 10 + da = xr.DataArray(np.arange(20).reshape(4, 5), dims=['y', 'x'], + coords={'lat': (('y', 'x'), lat), + 'lon': (('y', 'x'), lon)}) + + hdl = da.plot.line(x='lon', hue='x') + assert len(hdl) == 5 + + plt.clf() + hdl = da.plot.line(x='lon', hue='y') + assert len(hdl) == 4 + + with pytest.raises(ValueError, message='If x or y are 2D '): + da.plot.line(x='lon', hue='lat') + def test_2d_before_squeeze(self): a = DataArray(easy_array((1, 5))) a.plot() From 4bad455a801e91b329794895afa0040c868ff128 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 11 Oct 2018 14:20:52 -0700 Subject: [PATCH 248/282] Fix backend test classes so they actually run (#2479) * Fix backend test classes so they actually run We accidentally stopped running many backend tests when we merged GH2467, because they no longer inherited from untitest.TestCase and didn't have a name starting with "Test". This PR renames the tests so they actually run. I also fixed a bug with the pseudonetcdf backend that appears to have been introduced by a bad merge in GH2261. It wasn't caught because the tests weren't actually running. * Fixup test_conventions.py --- xarray/backends/pseudonetcdf_.py | 6 +-- xarray/tests/test_backends.py | 71 ++++++++++++++++---------------- xarray/tests/test_conventions.py | 4 +- 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index e4691d1f7e1..606ed5251ac 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -5,7 +5,7 @@ from .. import Variable from ..core import indexing from ..core.pycompat import OrderedDict -from ..core.utils import Frozen +from ..core.utils import Frozen, FrozenOrderedDict from .common import AbstractDataStore, BackendArray from .file_manager import CachingFileManager from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock @@ -73,8 +73,8 @@ def open_store_variable(self, name, var): return Variable(var.dimensions, data, attrs) def get_variables(self): - return ((k, self.open_store_variable(k, v)) - for k, v in self.ds.variables.items()) + return FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in self.ds.variables.items()) def get_attrs(self): return Frozen(dict([(k, getattr(self.ds, k)) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 43811942d5f..75aaba718c8 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -136,7 +136,7 @@ class NetCDF3Only(object): pass -class DatasetIOTestCases(object): +class DatasetIOBase(object): engine = None file_format = None @@ -593,7 +593,7 @@ def test_ondisk_after_print(self): assert not on_disk['var1']._in_memory -class CFEncodedDataTest(DatasetIOTestCases): +class CFEncodedBase(DatasetIOBase): def test_roundtrip_bytes_with_fill_value(self): values = np.array([b'ab', b'cdef', np.nan], dtype=object) @@ -895,7 +895,7 @@ def create_tmp_files(nfiles, suffix='.nc', allow_cleanup_failure=False): yield files -class BaseNetCDF4Test(CFEncodedDataTest): +class NetCDF4Base(CFEncodedBase): """Tests for both netCDF4-python and h5netcdf.""" engine = 'netcdf4' @@ -1177,7 +1177,7 @@ def test_read_variable_len_strings(self): @requires_netCDF4 -class NetCDF4DataTest(BaseNetCDF4Test): +class TestNetCDF4Data(NetCDF4Base): @contextlib.contextmanager def create_store(self): @@ -1254,11 +1254,11 @@ def test_autoclose_future_warning(self): @requires_netCDF4 @requires_dask -class NetCDF4ViaDaskDataTest(NetCDF4DataTest): +class TestNetCDF4ViaDaskData(TestNetCDF4Data): @contextlib.contextmanager def roundtrip(self, data, save_kwargs={}, open_kwargs={}, allow_cleanup_failure=False): - with NetCDF4DataTest.roundtrip( + with TestNetCDF4Data.roundtrip( self, data, save_kwargs, open_kwargs, allow_cleanup_failure) as ds: yield ds.chunk() @@ -1291,7 +1291,7 @@ def test_write_inconsistent_chunks(self): @requires_zarr -class BaseZarrTest(CFEncodedDataTest): +class ZarrBase(CFEncodedBase): DIMENSION_KEY = '_ARRAY_DIMENSIONS' @@ -1481,19 +1481,19 @@ def test_encoding_kwarg_fixed_width_string(self): # makes sense for Zarr backend @pytest.mark.xfail(reason="Zarr caching not implemented") def test_dataset_caching(self): - super(CFEncodedDataTest, self).test_dataset_caching() + super(CFEncodedBase, self).test_dataset_caching() @pytest.mark.xfail(reason="Zarr stores can not be appended to") def test_append_write(self): - super(CFEncodedDataTest, self).test_append_write() + super(CFEncodedBase, self).test_append_write() @pytest.mark.xfail(reason="Zarr stores can not be appended to") def test_append_overwrite_values(self): - super(CFEncodedDataTest, self).test_append_overwrite_values() + super(CFEncodedBase, self).test_append_overwrite_values() @pytest.mark.xfail(reason="Zarr stores can not be appended to") def test_append_with_invalid_dim_raises(self): - super(CFEncodedDataTest, self).test_append_with_invalid_dim_raises() + super(CFEncodedBase, self).test_append_with_invalid_dim_raises() def test_to_zarr_compute_false_roundtrip(self): from dask.delayed import Delayed @@ -1525,37 +1525,37 @@ def test_encoding_chunksizes(self): @requires_zarr -class ZarrDictStoreTest(BaseZarrTest): +class TestZarrDictStore(ZarrBase): @contextlib.contextmanager def create_zarr_target(self): yield {} @requires_zarr -class ZarrDirectoryStoreTest(BaseZarrTest): +class TestZarrDirectoryStore(ZarrBase): @contextlib.contextmanager def create_zarr_target(self): with create_tmp_file(suffix='.zarr') as tmp: yield tmp -class ScipyWriteTest(CFEncodedDataTest, NetCDF3Only): +class ScipyWriteBase(CFEncodedBase, NetCDF3Only): def test_append_write(self): import scipy if scipy.__version__ == '1.0.1': pytest.xfail('https://github.com/scipy/scipy/issues/8625') - super(ScipyWriteTest, self).test_append_write() + super(ScipyWriteBase, self).test_append_write() def test_append_overwrite_values(self): import scipy if scipy.__version__ == '1.0.1': pytest.xfail('https://github.com/scipy/scipy/issues/8625') - super(ScipyWriteTest, self).test_append_overwrite_values() + super(ScipyWriteBase, self).test_append_overwrite_values() @requires_scipy -class ScipyInMemoryDataTest(ScipyWriteTest): +class TestScipyInMemoryData(ScipyWriteBase): engine = 'scipy' @contextlib.contextmanager @@ -1576,7 +1576,7 @@ def test_bytes_pickle(self): @requires_scipy -class ScipyFileObjectTest(ScipyWriteTest): +class TestScipyFileObject(ScipyWriteBase): engine = 'scipy' @contextlib.contextmanager @@ -1604,7 +1604,7 @@ def test_pickle_dataarray(self): @requires_scipy -class ScipyFilePathTest(ScipyWriteTest): +class TestScipyFilePath(ScipyWriteBase): engine = 'scipy' @contextlib.contextmanager @@ -1641,7 +1641,7 @@ def test_nc4_scipy(self): @requires_netCDF4 -class NetCDF3ViaNetCDF4DataTest(CFEncodedDataTest, NetCDF3Only): +class TestNetCDF3ViaNetCDF4Data(CFEncodedBase, NetCDF3Only): engine = 'netcdf4' file_format = 'NETCDF3_CLASSIC' @@ -1661,8 +1661,7 @@ def test_encoding_kwarg_vlen_string(self): @requires_netCDF4 -class NetCDF4ClassicViaNetCDF4DataTest(CFEncodedDataTest, NetCDF3Only, - object): +class TestNetCDF4ClassicViaNetCDF4Data(CFEncodedBase, NetCDF3Only): engine = 'netcdf4' file_format = 'NETCDF4_CLASSIC' @@ -1675,7 +1674,7 @@ def create_store(self): @requires_scipy_or_netCDF4 -class GenericNetCDFDataTest(CFEncodedDataTest, NetCDF3Only): +class TestGenericNetCDFData(CFEncodedBase, NetCDF3Only): # verify that we can read and write netCDF3 files as long as we have scipy # or netCDF4-python installed file_format = 'netcdf3_64bit' @@ -1752,7 +1751,7 @@ def test_encoding_unlimited_dims(self): @requires_h5netcdf @requires_netCDF4 -class H5NetCDFDataTest(BaseNetCDF4Test): +class TestH5NetCDFData(NetCDF4Base): engine = 'h5netcdf' @contextlib.contextmanager @@ -1955,7 +1954,7 @@ def test_open_mfdataset_manyfiles(readengine, nfiles, parallel, chunks, @requires_scipy_or_netCDF4 -class OpenMFDatasetWithDataVarsAndCoordsKwTest(object): +class TestOpenMFDatasetWithDataVarsAndCoordsKw(object): coord_name = 'lon' var_name = 'v1' @@ -2063,7 +2062,7 @@ def test_invalid_data_vars_value_should_fail(self): @requires_dask @requires_scipy @requires_netCDF4 -class DaskTest(DatasetIOTestCases): +class TestDask(DatasetIOBase): @contextlib.contextmanager def create_store(self): yield Dataset() @@ -2073,7 +2072,7 @@ def roundtrip(self, data, save_kwargs={}, open_kwargs={}, allow_cleanup_failure=False): yield data.chunk() - # Override methods in DatasetIOTestCases - not applicable to dask + # Override methods in DatasetIOBase - not applicable to dask def test_roundtrip_string_encoded_characters(self): pass @@ -2081,7 +2080,7 @@ def test_roundtrip_coordinates_with_space(self): pass def test_roundtrip_numpy_datetime_data(self): - # Override method in DatasetIOTestCases - remove not applicable + # Override method in DatasetIOBase - remove not applicable # save_kwds times = pd.to_datetime(['2000-01-01', '2000-01-02', 'NaT']) expected = Dataset({'t': ('t', times), 't0': times[0]}) @@ -2089,7 +2088,7 @@ def test_roundtrip_numpy_datetime_data(self): assert_identical(expected, actual) def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): - # Override method in DatasetIOTestCases - remove not applicable + # Override method in DatasetIOBase - remove not applicable # save_kwds from .test_coding_times import _all_cftime_date_types @@ -2109,7 +2108,7 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): assert (abs_diff <= np.timedelta64(1, 's')).all() def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): - # Override method in DatasetIOTestCases - remove not applicable + # Override method in DatasetIOBase - remove not applicable # save_kwds from .test_coding_times import _all_cftime_date_types @@ -2129,7 +2128,7 @@ def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): assert (abs_diff <= np.timedelta64(1, 's')).all() def test_write_store(self): - # Override method in DatasetIOTestCases - not applicable to dask + # Override method in DatasetIOBase - not applicable to dask pass def test_dataset_caching(self): @@ -2312,7 +2311,7 @@ def test_deterministic_names(self): def test_dataarray_compute(self): # Test DataArray.compute() on dask backend. - # The test for Dataset.compute() is already in DatasetIOTestCases; + # The test for Dataset.compute() is already in DatasetIOBase; # however dask is the only tested backend which supports DataArrays actual = DataArray([1, 2]).chunk() computed = actual.compute() @@ -2338,7 +2337,7 @@ def test_save_mfdataset_compute_false_roundtrip(self): @requires_scipy_or_netCDF4 @requires_pydap -class PydapTest(object): +class TestPydap(object): def convert_to_pydap_dataset(self, original): from pydap.model import GridType, BaseType, DatasetType ds = DatasetType('bears', **original.attrs) @@ -2418,7 +2417,7 @@ def test_dask(self): @network @requires_scipy_or_netCDF4 @requires_pydap -class PydapOnlineTest(PydapTest): +class TestPydapOnline(TestPydap): @contextlib.contextmanager def create_datasets(self, **kwargs): url = 'http://test.opendap.org/opendap/hyrax/data/nc/bears.nc' @@ -2439,7 +2438,7 @@ def test_session(self): @requires_scipy @requires_pynio -class PyNioTest(ScipyWriteTest): +class TestPyNio(ScipyWriteBase): def test_write_store(self): # pynio is read-only for now pass @@ -2466,7 +2465,7 @@ def test_weakrefs(self): @requires_pseudonetcdf @pytest.mark.filterwarnings('ignore:IOAPI_ISPH is assumed to be 6370000') -class PseudoNetCDFFormatTest(object): +class TestPseudoNetCDFFormat(object): def open(self, path, **kwargs): return open_dataset(path, engine='pseudonetcdf', **kwargs) diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index a067d01a308..5fa518f5112 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -18,7 +18,7 @@ from . import ( assert_array_equal, raises_regex, requires_cftime_or_netCDF4, requires_dask, requires_netCDF4) -from .test_backends import CFEncodedDataTest +from .test_backends import CFEncodedBase class TestBoolTypeArray(object): @@ -255,7 +255,7 @@ def encode_variable(self, var): @requires_netCDF4 -class TestCFEncodedDataStore(CFEncodedDataTest): +class TestCFEncodedDataStore(CFEncodedBase): @contextlib.contextmanager def create_store(self): yield CFEncodedInMemoryStore() From 7cab33a1335cc2cbeb93090145a7f6d4c25a1692 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Wed, 17 Oct 2018 00:00:56 -0400 Subject: [PATCH 249/282] Improve arithmetic operations involving CFTimeIndexes and TimedeltaIndexes (#2485) * Improve arithmetic involving CFTimeIndexes and TimedeltaIndexes * Fix Appveyor Python 2.7 test failure * Clarify comment in _add_delta --- doc/whats-new.rst | 9 ++++++- xarray/coding/cftimeindex.py | 16 +++++++++++- xarray/tests/test_cftimeindex.py | 44 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index c8ae5ac43c8..de7e6c8f6ff 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -71,7 +71,7 @@ Enhancements :py:meth:`~xarray.Dataset.differentiate`, :py:meth:`~xarray.DataArray.interp`, and :py:meth:`~xarray.Dataset.interp`. - By `Spencer Clark `_ + By `Spencer Clark `_. Bug fixes ~~~~~~~~~ @@ -97,6 +97,13 @@ Bug fixes ``open_rasterio`` to error (:issue:`2454`). By `Stephan Hoyer `_. +- Subtracting one CFTimeIndex from another now returns a + ``pandas.TimedeltaIndex``, analogous to the behavior for DatetimeIndexes + (:issue:`2484`). By `Spencer Clark `_. +- Adding a TimedeltaIndex to, or subtracting a TimedeltaIndex from a + CFTimeIndex is now allowed (:issue:`2484`). + By `Spencer Clark `_. + .. _whats-new.0.10.9: v0.10.9 (21 September 2018) diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index dea896c199a..5de055c1b9a 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -359,13 +359,27 @@ def shift(self, n, freq): "str or datetime.timedelta, got {}.".format(freq)) def __add__(self, other): + if isinstance(other, pd.TimedeltaIndex): + other = other.to_pytimedelta() return CFTimeIndex(np.array(self) + other) def __radd__(self, other): + if isinstance(other, pd.TimedeltaIndex): + other = other.to_pytimedelta() return CFTimeIndex(other + np.array(self)) def __sub__(self, other): - return CFTimeIndex(np.array(self) - other) + if isinstance(other, CFTimeIndex): + return pd.TimedeltaIndex(np.array(self) - np.array(other)) + elif isinstance(other, pd.TimedeltaIndex): + return CFTimeIndex(np.array(self) - other.to_pytimedelta()) + else: + return CFTimeIndex(np.array(self) - other) + + def _add_delta(self, deltas): + # To support TimedeltaIndex + CFTimeIndex with older versions of + # pandas. No longer used as of pandas 0.23. + return self + deltas def _parse_iso8601_without_reso(date_type, datetime_str): diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index d1726ab3313..e18c55d2fae 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -629,6 +629,17 @@ def test_cftimeindex_add(index): assert isinstance(result, CFTimeIndex) +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _CFTIME_CALENDARS) +def test_cftimeindex_add_timedeltaindex(calendar): + a = xr.cftime_range('2000', periods=5, calendar=calendar) + deltas = pd.TimedeltaIndex([timedelta(days=2) for _ in range(5)]) + result = a + deltas + expected = a.shift(2, 'D') + assert result.equals(expected) + assert isinstance(result, CFTimeIndex) + + @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_cftimeindex_radd(index): date_type = index.date_type @@ -640,6 +651,17 @@ def test_cftimeindex_radd(index): assert isinstance(result, CFTimeIndex) +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _CFTIME_CALENDARS) +def test_timedeltaindex_add_cftimeindex(calendar): + a = xr.cftime_range('2000', periods=5, calendar=calendar) + deltas = pd.TimedeltaIndex([timedelta(days=2) for _ in range(5)]) + result = deltas + a + expected = a.shift(2, 'D') + assert result.equals(expected) + assert isinstance(result, CFTimeIndex) + + @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_cftimeindex_sub(index): date_type = index.date_type @@ -652,6 +674,28 @@ def test_cftimeindex_sub(index): assert isinstance(result, CFTimeIndex) +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _CFTIME_CALENDARS) +def test_cftimeindex_sub_cftimeindex(calendar): + a = xr.cftime_range('2000', periods=5, calendar=calendar) + b = a.shift(2, 'D') + result = b - a + expected = pd.TimedeltaIndex([timedelta(days=2) for _ in range(5)]) + assert result.equals(expected) + assert isinstance(result, pd.TimedeltaIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _CFTIME_CALENDARS) +def test_cftimeindex_sub_timedeltaindex(calendar): + a = xr.cftime_range('2000', periods=5, calendar=calendar) + deltas = pd.TimedeltaIndex([timedelta(days=2) for _ in range(5)]) + result = a - deltas + expected = a.shift(-2, 'D') + assert result.equals(expected) + assert isinstance(result, CFTimeIndex) + + @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_cftimeindex_rsub(index): with pytest.raises(TypeError): From 9f4474d657193f1c7c9aac25bb2edf94755a8593 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Wed, 17 Oct 2018 18:53:31 +0200 Subject: [PATCH 250/282] Add a GRIB backend via ECMWF cfgrib / ecCodes (#2476) * Integration of ECMWF cfgrib driver to read GRIB files into xarray. * Remove all coordinate renaming from the cfgrib backend. * Move flavour selection to `cfgrib.Dataset.from_path`. * Sync xarray backend import style with xarray. * Make use of the new xarray.backends.FileCachingManager. * Add just-in-case locking for ecCodes. * Explicitly assign attributes to CfGribArrayWrapper * Add missing locking in CfGribArrayWrapper and use explicit_indexing_adapter. * Add a comment about the ugly work-around needed for filter_by_keys. * Declare correct indexing support. * Add TestCfGrib test class. * cfgrib doesn't store a file reference so no need for CachingFileManager. * Add cfgrib testing to Travis-CI. * Naming. * Fix line lengths and get to 100% coverage. * Add reference to *cfgrib* engine in inline docs. * First cut of the documentation. * Tentative test cfgrib under dask.distributed. * Better integration test. * Remove explicit copyright and license boilerplate to harmonise with other files. * Add a usage example. * Fix code style. * Fix doc style. * Fix docs testing. The example.grib file is not accessible. * Fix merge in docs. * Fix merge in docs. * Fix doc style. --- ci/requirements-py36.yml | 2 + doc/installing.rst | 4 +- doc/io.rst | 22 +++++++++ doc/whats-new.rst | 6 ++- xarray/backends/__init__.py | 2 + xarray/backends/api.py | 12 +++-- xarray/backends/cfgrib_.py | 78 +++++++++++++++++++++++++++++++ xarray/tests/__init__.py | 1 + xarray/tests/data/example.grib | Bin 0 -> 5232 bytes xarray/tests/test_backends.py | 24 +++++++++- xarray/tests/test_distributed.py | 19 +++++++- 11 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 xarray/backends/cfgrib_.py create mode 100644 xarray/tests/data/example.grib diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index fd63fe26130..fc272984237 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -21,8 +21,10 @@ dependencies: - bottleneck - zarr - pseudonetcdf>=3.0.1 + - eccodes - pip: - coveralls - pytest-cov - pydap - lxml + - cfgrib diff --git a/doc/installing.rst b/doc/installing.rst index eb74eb7162b..64751eea637 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -34,7 +34,9 @@ For netCDF and IO - `rasterio `__: for reading GeoTiffs and other gridded raster datasets. - `iris `__: for conversion to and from iris' - Cube objects. + Cube objects +- `cfgrib `__: for reading GRIB files via the + *ECMWF ecCodes* library. For accelerating xarray ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/io.rst b/doc/io.rst index 093ee773e15..e841e665308 100644 --- a/doc/io.rst +++ b/doc/io.rst @@ -635,6 +635,28 @@ For example: Not all native zarr compression and filtering options have been tested with xarray. +.. _io.cfgrib: + +GRIB format via cfgrib +---------------------- + +xarray supports reading GRIB files via ECMWF cfgrib_ python driver and ecCodes_ +C-library, if they are installed. To open a GRIB file supply ``engine='cfgrib'`` +to :py:func:`~xarray.open_dataset`: + +.. ipython:: + :verbatim: + + In [1]: ds_grib = xr.open_dataset('example.grib', engine='cfgrib') + +We recommend installing ecCodes via conda:: + + conda install -c conda-forge eccodes + pip install cfgrib + +.. _cfgrib: https://github.com/ecmwf/cfgrib +.. _ecCodes: https://confluence.ecmwf.int/display/ECC/ecCodes+Home + .. _io.pynio: Formats supported by PyNIO diff --git a/doc/whats-new.rst b/doc/whats-new.rst index de7e6c8f6ff..61da801badb 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -72,7 +72,11 @@ Enhancements :py:meth:`~xarray.DataArray.interp`, and :py:meth:`~xarray.Dataset.interp`. By `Spencer Clark `_. - +- Added a new backend for the GRIB file format based on ECMWF *cfgrib* + python driver and *ecCodes* C-library. (:issue:`2475`) + By `Alessandro Amici `_, + sponsored by `ECMWF `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index a2f0d79a6d1..9b9e04d9346 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -5,6 +5,7 @@ """ from .common import AbstractDataStore from .file_manager import FileManager, CachingFileManager, DummyFileManager +from .cfgrib_ import CfGribDataStore from .memory import InMemoryDataStore from .netCDF4_ import NetCDF4DataStore from .pydap_ import PydapDataStore @@ -18,6 +19,7 @@ 'AbstractDataStore', 'FileManager', 'CachingFileManager', + 'CfGribDataStore', 'DummyFileManager', 'InMemoryDataStore', 'NetCDF4DataStore', diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 65112527045..3fb7338e171 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -162,7 +162,8 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, decode_coords : bool, optional If True, decode the 'coordinates' attribute to identify coordinates in the resulting dataset. - engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', 'pseudonetcdf'}, optional + engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', 'cfgrib', + 'pseudonetcdf'}, optional Engine to use when reading files. If not provided, the default engine is chosen based on available dependencies, with a preference for 'netcdf4'. @@ -296,6 +297,9 @@ def maybe_decode_store(store, lock=False): elif engine == 'pseudonetcdf': store = backends.PseudoNetCDFDataStore.open( filename_or_obj, lock=lock, **backend_kwargs) + elif engine == 'cfgrib': + store = backends.CfGribDataStore( + filename_or_obj, lock=lock, **backend_kwargs) else: raise ValueError('unrecognized engine for open_dataset: %r' % engine) @@ -356,7 +360,8 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, decode_coords : bool, optional If True, decode the 'coordinates' attribute to identify coordinates in the resulting dataset. - engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio'}, optional + engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', 'cfgrib'}, + optional Engine to use when reading files. If not provided, the default engine is chosen based on available dependencies, with a preference for 'netcdf4'. @@ -486,7 +491,8 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, of all non-null values. preprocess : callable, optional If provided, call this function on each dataset prior to concatenation. - engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio'}, optional + engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', 'cfgrib'}, + optional Engine to use when reading files. If not provided, the default engine is chosen based on available dependencies, with a preference for 'netcdf4'. diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py new file mode 100644 index 00000000000..c0a7c025606 --- /dev/null +++ b/xarray/backends/cfgrib_.py @@ -0,0 +1,78 @@ +from __future__ import absolute_import, division, print_function + +import numpy as np + +from .. import Variable +from ..core import indexing +from ..core.utils import Frozen, FrozenOrderedDict +from .common import AbstractDataStore, BackendArray +from .locks import ensure_lock, SerializableLock + +# FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe +# in most circumstances. See: +# https://confluence.ecmwf.int/display/ECC/Frequently+Asked+Questions +ECCODES_LOCK = SerializableLock() + + +class CfGribArrayWrapper(BackendArray): + def __init__(self, datastore, array): + self.datastore = datastore + self.shape = array.shape + self.dtype = array.dtype + self.array = array + + def __getitem__(self, key): + return indexing.explicit_indexing_adapter( + key, self.shape, indexing.IndexingSupport.OUTER, self._getitem) + + def _getitem(self, key): + with self.datastore.lock: + return self.array[key] + + +class CfGribDataStore(AbstractDataStore): + """ + Implements the ``xr.AbstractDataStore`` read-only API for a GRIB file. + """ + def __init__(self, filename, lock=None, **backend_kwargs): + import cfgrib + if lock is None: + lock = ECCODES_LOCK + self.lock = ensure_lock(lock) + + # NOTE: filter_by_keys is a dict, but CachingFileManager only accepts + # hashable types. + if 'filter_by_keys' in backend_kwargs: + filter_by_keys_items = backend_kwargs['filter_by_keys'].items() + backend_kwargs['filter_by_keys'] = tuple(filter_by_keys_items) + + self.ds = cfgrib.open_file(filename, mode='r', **backend_kwargs) + + def open_store_variable(self, name, var): + if isinstance(var.data, np.ndarray): + data = var.data + else: + wrapped_array = CfGribArrayWrapper(self, var.data) + data = indexing.LazilyOuterIndexedArray(wrapped_array) + + encoding = self.ds.encoding.copy() + encoding['original_shape'] = var.data.shape + + return Variable(var.dimensions, data, var.attributes, encoding) + + def get_variables(self): + return FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in self.ds.variables.items()) + + def get_attrs(self): + return Frozen(self.ds.attributes) + + def get_dimensions(self): + return Frozen(self.ds.dimensions) + + def get_encoding(self): + dims = self.get_dimensions() + encoding = { + 'unlimited_dims': {k for k, v in dims.items() if v is None}, + } + return encoding diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 5f724dd6713..56ecfa30c4d 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -77,6 +77,7 @@ def LooseVersion(vstring): has_zarr, requires_zarr = _importorskip('zarr', minversion='2.2') has_np113, requires_np113 = _importorskip('numpy', minversion='1.13.0') has_iris, requires_iris = _importorskip('iris') +has_cfgrib, requires_cfgrib = _importorskip('cfgrib') # some special cases has_scipy_or_netCDF4 = has_scipy or has_netCDF4 diff --git a/xarray/tests/data/example.grib b/xarray/tests/data/example.grib new file mode 100644 index 0000000000000000000000000000000000000000..596a54d98a022568e41ebe83eedb94d6561a09a6 GIT binary patch literal 5232 zcmds)F=*6K9LB%2*DFP(hah%v2vSiux84}6cR5do;&g+>41(0j&DM5!4#FuNI*4@W zkins2Q7Cj1p-}ExXme)=7f%Oq5Uk+PL5csDt1rpLkQeN|YYpEe?_M5Be)s?0_j2Z~ z>sMVDeQ0m*>2OAa3uK@Te$+JH%};bT^ag#1NfPN$ugtwxblP=xw9V_ z@A%Vsfe+RCGjRDB@A7y1@c6ZUa82r@Cm5g|-K$vG2BYH$@62ZDsc?DFnt;%=9A+5q zGtBPsAz~Y&%}gNUEWIN$e}J9pAX_vb^i#)f3r!2(n4sbf2{Nyiy(25nN69922z_~j z2E(e2hLM&`ksEK`F#hI7v-k?^L`ThrrXl29LcN8p1=RDY$<(+pj?6qCCDSSp+N@xG z88sUlD{09TxiOBc@Ht8rM0i()GYw}O%{+#6UlQ|7ksJMN^lm7!v3M<-jduW{rxSRX z!_*#Tj3BT~f@o^o7)MqbjV8zo9YVi=9Rp1R-|e(yirg4SR-TKJ-EKkXtA&PzH49Gq zn(?2a`>F5GvydCj;@jvv8v=xOR#1O{Er!~D+L|eHV;q@zF-oT8A+(vtdKI+_HkQ+p zDRN^RS>aQZEQqjM0+r3#N3(_D5nsK_MtVA#_r~!#fSvAGse@!oz#>5PTJQ$Z`(;}) zWZlU|S+dy#G8<^a5ud=Kh6Guc+;~`Ighg;Ky~>>fge1tiv{HAH`SXm<`2S+YrxI#b z1?D|0a-+bah76e+H~s_JbAUvADajPMF^=qBLd~dDgk^hW8wG@9YR1iR_$xJCG$tEE zE45@c(mXo`*%&61%SNghRqm)~W0>qwf*Xm6L^WFi^ni|z8~u3}Ci|bVF*O;@Fj}UM TK{kfTq_dIcnFvN|a5w%2m#N8p literal 0 HcmV?d00001 diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 75aaba718c8..a274b41c424 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -33,7 +33,7 @@ has_dask, has_netCDF4, has_scipy, network, raises_regex, requires_cftime, requires_dask, requires_h5netcdf, requires_netCDF4, requires_pathlib, requires_pseudonetcdf, requires_pydap, requires_pynio, requires_rasterio, - requires_scipy, requires_scipy_or_netCDF4, requires_zarr) + requires_scipy, requires_scipy_or_netCDF4, requires_zarr, requires_cfgrib) from .test_dataset import create_test_data try: @@ -2463,6 +2463,28 @@ def test_weakrefs(self): assert_identical(actual, expected) +@requires_cfgrib +class TestCfGrib(object): + + def test_read(self): + expected = {'number': 2, 'time': 3, 'air_pressure': 2, 'latitude': 3, + 'longitude': 4} + with open_example_dataset('example.grib', engine='cfgrib') as ds: + assert ds.dims == expected + assert list(ds.data_vars) == ['z', 't'] + assert ds['z'].min() == 12660. + + def test_read_filter_by_keys(self): + kwargs = {'filter_by_keys': {'shortName': 't'}} + expected = {'number': 2, 'time': 3, 'air_pressure': 2, 'latitude': 3, + 'longitude': 4} + with open_example_dataset('example.grib', engine='cfgrib', + backend_kwargs=kwargs) as ds: + assert ds.dims == expected + assert list(ds.data_vars) == ['t'] + assert ds['t'].min() == 231. + + @requires_pseudonetcdf @pytest.mark.filterwarnings('ignore:IOAPI_ISPH is assumed to be 6370000') class TestPseudoNetCDFFormat(object): diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 7c77a62d3c9..1837a0fe4ef 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -20,12 +20,13 @@ import xarray as xr from xarray.backends.locks import HDF5_LOCK, CombinedLock from xarray.tests.test_backends import (ON_WINDOWS, create_tmp_file, - create_tmp_geotiff) + create_tmp_geotiff, + open_example_dataset) from xarray.tests.test_dataset import create_test_data from . import ( assert_allclose, has_h5netcdf, has_netCDF4, requires_rasterio, has_scipy, - requires_zarr, raises_regex) + requires_zarr, requires_cfgrib, raises_regex) # this is to stop isort throwing errors. May have been easier to just use # `isort:skip` in retrospect @@ -142,6 +143,20 @@ def test_dask_distributed_rasterio_integration_test(loop): assert_allclose(actual, expected) +@requires_cfgrib +def test_dask_distributed_cfgrib_integration_test(loop): + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: + with open_example_dataset('example.grib', + engine='cfgrib', + chunks={'time': 1}) as ds: + with open_example_dataset('example.grib', + engine='cfgrib') as expected: + assert isinstance(ds['t'].data, da.Array) + actual = ds.compute() + assert_allclose(actual, expected) + + @pytest.mark.skipif(distributed.__version__ <= '1.19.3', reason='Need recent distributed version to clean up get') @gen_cluster(client=True, timeout=None) From dffbcb8b7a6745ddbb061d554c0475afb9122bf3 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Mon, 22 Oct 2018 02:01:03 +0200 Subject: [PATCH 251/282] Small fix in rasterio docs (#2498) --- xarray/backends/rasterio_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 5746b4e748d..7a343a6529e 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -169,7 +169,7 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, from affine import Affine da = xr.open_rasterio('path_to_file.tif') - transform = Affine(*da.attrs['transform']) + transform = Affine.from_gdal(*da.attrs['transform']) nx, ny = da.sizes['x'], da.sizes['y'] x, y = np.meshgrid(np.arange(nx)+0.5, np.arange(ny)+0.5) * transform From 671d936166414edc368cca8e33475369e2bb4d24 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Mon, 22 Oct 2018 17:26:13 +0200 Subject: [PATCH 252/282] ENH: Detect the GRIB files by the filename extension and suggest engine. (#2492) * Detect the GRIB files by the filename extension. * Follow conventions on QA and coverage. * Add test_backends_api.py to unit test private function in api.py. * Add tests to is_grib_path and is_remote_uri in utils.py * Move `.gz` detection logic to `_get_default_engine`. * More expressive test. * Do not fall back to other inconsistent drivers for GRIB files. * Fix string split. * "In the face of ambiguity, refuse the temptation to guess." * Move import testing for every class of input to a different function. * Fix _get_default_engine test. --- xarray/backends/api.py | 89 +++++++++++++++++++++---------- xarray/core/utils.py | 6 +++ xarray/tests/test_backends_api.py | 22 ++++++++ xarray/tests/test_utils.py | 16 ++++++ 4 files changed, 106 insertions(+), 27 deletions(-) create mode 100644 xarray/tests/test_backends_api.py diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 3fb7338e171..ca440872d73 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -12,7 +12,7 @@ from ..core import indexing from ..core.combine import auto_combine from ..core.pycompat import basestring, path_type -from ..core.utils import close_on_error, is_remote_uri +from ..core.utils import close_on_error, is_remote_uri, is_grib_path from .common import ArrayWriter from .locks import _get_scheduler @@ -21,29 +21,71 @@ DATAARRAY_VARIABLE = '__xarray_dataarray_variable__' -def _get_default_engine(path, allow_remote=False): - if allow_remote and is_remote_uri(path): # pragma: no cover +def _get_default_engine_remote_uri(): + try: + import netCDF4 + engine = 'netcdf4' + except ImportError: # pragma: no cover try: - import netCDF4 - engine = 'netcdf4' + import pydap # flake8: noqa + engine = 'pydap' except ImportError: - try: - import pydap # flake8: noqa - engine = 'pydap' - except ImportError: - raise ValueError('netCDF4 or pydap is required for accessing ' - 'remote datasets via OPeNDAP') + raise ValueError('netCDF4 or pydap is required for accessing ' + 'remote datasets via OPeNDAP') + return engine + + +def _get_default_engine_grib(): + msgs = [] + try: + import Nio # flake8: noqa + msgs += ["set engine='pynio' to access GRIB files with PyNIO"] + except ImportError: # pragma: no cover + pass + try: + import cfgrib # flake8: noqa + msgs += ["set engine='cfgrib' to access GRIB files with cfgrib"] + except ImportError: # pragma: no cover + pass + if msgs: + raise ValueError(' or\n'.join(msgs)) else: + raise ValueError('PyNIO or cfgrib is required for accessing ' + 'GRIB files') + + +def _get_default_engine_gz(): + try: + import scipy # flake8: noqa + engine = 'scipy' + except ImportError: # pragma: no cover + raise ValueError('scipy is required for accessing .gz files') + return engine + + +def _get_default_engine_netcdf(): + try: + import netCDF4 # flake8: noqa + engine = 'netcdf4' + except ImportError: # pragma: no cover try: - import netCDF4 # flake8: noqa - engine = 'netcdf4' - except ImportError: # pragma: no cover - try: - import scipy.io.netcdf # flake8: noqa - engine = 'scipy' - except ImportError: - raise ValueError('cannot read or write netCDF files without ' - 'netCDF4-python or scipy installed') + import scipy.io.netcdf # flake8: noqa + engine = 'scipy' + except ImportError: + raise ValueError('cannot read or write netCDF files without ' + 'netCDF4-python or scipy installed') + return engine + + +def _get_default_engine(path, allow_remote=False): + if allow_remote and is_remote_uri(path): + engine = _get_default_engine_remote_uri() + elif is_grib_path(path): + engine = _get_default_engine_grib() + elif path.endswith('.gz'): + engine = _get_default_engine_gz() + else: + engine = _get_default_engine_netcdf() return engine @@ -270,13 +312,6 @@ def maybe_decode_store(store, lock=False): elif isinstance(filename_or_obj, basestring): filename_or_obj = _normalize_path(filename_or_obj) - if filename_or_obj.endswith('.gz'): - if engine is not None and engine != 'scipy': - raise ValueError('can only read gzipped netCDF files with ' - "default engine or engine='scipy'") - else: - engine = 'scipy' - if engine is None: engine = _get_default_engine(filename_or_obj, allow_remote=True) diff --git a/xarray/core/utils.py b/xarray/core/utils.py index c39a07e1b5a..5c9d8bfbf77 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -5,6 +5,7 @@ import contextlib import functools import itertools +import os.path import re import warnings from collections import Iterable, Mapping, MutableMapping, MutableSet @@ -504,6 +505,11 @@ def is_remote_uri(path): return bool(re.search('^https?\://', path)) +def is_grib_path(path): + _, ext = os.path.splitext(path) + return ext in ['.grib', '.grb', '.grib2', '.grb2'] + + def is_uniform_spaced(arr, **kwargs): """Return True if values of an array are uniformly spaced and sorted. diff --git a/xarray/tests/test_backends_api.py b/xarray/tests/test_backends_api.py new file mode 100644 index 00000000000..ed49dd721d2 --- /dev/null +++ b/xarray/tests/test_backends_api.py @@ -0,0 +1,22 @@ + +import pytest + +from xarray.backends.api import _get_default_engine +from . import requires_netCDF4, requires_scipy + + +@requires_netCDF4 +@requires_scipy +def test__get_default_engine(): + engine_remote = _get_default_engine('http://example.org/test.nc', + allow_remote=True) + assert engine_remote == 'netcdf4' + + engine_gz = _get_default_engine('/example.gz') + assert engine_gz == 'scipy' + + with pytest.raises(ValueError): + _get_default_engine('/example.grib') + + engine_default = _get_default_engine('/example') + assert engine_default == 'netcdf4' diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 34f401dd243..33021fc5ef4 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -200,6 +200,22 @@ def test_repr_object(): assert repr(obj) == 'foo' +def test_is_remote_uri(): + assert utils.is_remote_uri('http://example.com') + assert utils.is_remote_uri('https://example.com') + assert not utils.is_remote_uri(' http://example.com') + assert not utils.is_remote_uri('example.nc') + + +def test_is_grib_path(): + assert not utils.is_grib_path('example.nc') + assert not utils.is_grib_path('example.grib ') + assert utils.is_grib_path('example.grib') + assert utils.is_grib_path('example.grib2') + assert utils.is_grib_path('example.grb') + assert utils.is_grib_path('example.grb2') + + class Test_is_uniform_and_sorted(object): def test_sorted_uniform(self): From 4a5d88dba5fd50d48ab00ed2ebaee058287ab0bf Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Mon, 22 Oct 2018 20:22:50 -0400 Subject: [PATCH 253/282] Avoid use of deprecated get= parameter in tests (#2500) * Update tests to use the scheduler= keyword rather than get= The get= keyword has been deprecated and will be removed in a future version. * replace use of get= with scheduler= in asv * add whats new entry --- asv_bench/benchmarks/dataset_io.py | 24 ++++++++++++------------ doc/whats-new.rst | 4 +++- xarray/tests/__init__.py | 2 +- xarray/tests/test_dask.py | 12 +++++++++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/asv_bench/benchmarks/dataset_io.py b/asv_bench/benchmarks/dataset_io.py index da18d541a16..3e070e1355b 100644 --- a/asv_bench/benchmarks/dataset_io.py +++ b/asv_bench/benchmarks/dataset_io.py @@ -168,7 +168,7 @@ def time_load_dataset_netcdf4_with_block_chunks_vindexing(self): ds = ds.isel(**self.vinds).load() def time_load_dataset_netcdf4_with_block_chunks_multiprocessing(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_dataset(self.filepath, engine='netcdf4', chunks=self.block_chunks).load() @@ -177,7 +177,7 @@ def time_load_dataset_netcdf4_with_time_chunks(self): chunks=self.time_chunks).load() def time_load_dataset_netcdf4_with_time_chunks_multiprocessing(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_dataset(self.filepath, engine='netcdf4', chunks=self.time_chunks).load() @@ -194,7 +194,7 @@ def setup(self): self.ds.to_netcdf(self.filepath, format=self.format) def time_load_dataset_scipy_with_block_chunks(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_dataset(self.filepath, engine='scipy', chunks=self.block_chunks).load() @@ -209,7 +209,7 @@ def time_load_dataset_scipy_with_block_chunks_vindexing(self): ds = ds.isel(**self.vinds).load() def time_load_dataset_scipy_with_time_chunks(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_dataset(self.filepath, engine='scipy', chunks=self.time_chunks).load() @@ -349,7 +349,7 @@ def time_load_dataset_netcdf4_with_block_chunks(self): chunks=self.block_chunks).load() def time_load_dataset_netcdf4_with_block_chunks_multiprocessing(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='netcdf4', chunks=self.block_chunks).load() @@ -358,7 +358,7 @@ def time_load_dataset_netcdf4_with_time_chunks(self): chunks=self.time_chunks).load() def time_load_dataset_netcdf4_with_time_chunks_multiprocessing(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='netcdf4', chunks=self.time_chunks).load() @@ -367,7 +367,7 @@ def time_open_dataset_netcdf4_with_block_chunks(self): chunks=self.block_chunks) def time_open_dataset_netcdf4_with_block_chunks_multiprocessing(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='netcdf4', chunks=self.block_chunks) @@ -376,7 +376,7 @@ def time_open_dataset_netcdf4_with_time_chunks(self): chunks=self.time_chunks) def time_open_dataset_netcdf4_with_time_chunks_multiprocessing(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='netcdf4', chunks=self.time_chunks) @@ -392,22 +392,22 @@ def setup(self): format=self.format) def time_load_dataset_scipy_with_block_chunks(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='scipy', chunks=self.block_chunks).load() def time_load_dataset_scipy_with_time_chunks(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='scipy', chunks=self.time_chunks).load() def time_open_dataset_scipy_with_block_chunks(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='scipy', chunks=self.block_chunks) def time_open_dataset_scipy_with_time_chunks(self): - with dask.set_options(get=dask.multiprocessing.get): + with dask.config.set(scheduler="multiprocessing"): xr.open_mfdataset(self.filenames_list, engine='scipy', chunks=self.time_chunks) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 61da801badb..9a55b9380b9 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -106,7 +106,9 @@ Bug fixes (:issue:`2484`). By `Spencer Clark `_. - Adding a TimedeltaIndex to, or subtracting a TimedeltaIndex from a CFTimeIndex is now allowed (:issue:`2484`). - By `Spencer Clark `_. + By `Spencer Clark `_. +- Avoid use of Dask's deprecated ``get=`` parameter in tests + by `Matthew Rocklin `_. .. _whats-new.0.10.9: diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 56ecfa30c4d..a45f71bbc3b 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -93,7 +93,7 @@ def LooseVersion(vstring): if LooseVersion(dask.__version__) < '0.18': dask.set_options(get=dask.get) else: - dask.config.set(scheduler='sync') + dask.config.set(scheduler='single-threaded') try: import_seaborn() has_seaborn = True diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index e56f751bef9..62ce7d074fa 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -26,7 +26,7 @@ class DaskTestCase(object): def assertLazyAnd(self, expected, actual, test): - with (dask.config.set(get=dask.get) + with (dask.config.set(scheduler='single-threaded') if LooseVersion(dask.__version__) >= LooseVersion('0.18.0') else dask.set_options(get=dask.get)): test(actual, expected) @@ -456,7 +456,11 @@ def counting_get(*args, **kwargs): count[0] += 1 return dask.get(*args, **kwargs) - ds.load(get=counting_get) + if dask.__version__ < '0.19.4': + ds.load(get=counting_get) + else: + ds.load(scheduler=counting_get) + assert count[0] == 1 def test_stack(self): @@ -831,7 +835,9 @@ def test_basic_compute(): dask.multiprocessing.get, dask.local.get_sync, None]: - with (dask.config.set(get=get) + with (dask.config.set(scheduler=get) + if LooseVersion(dask.__version__) >= LooseVersion('0.19.4') + else dask.config.set(scheduler=get) if LooseVersion(dask.__version__) >= LooseVersion('0.18.0') else dask.set_options(get=get)): ds.compute() From f9f4903ad15f10e65e0fb8526712062a18d145cb Mon Sep 17 00:00:00 2001 From: nedclimaterisk <43126798+nedclimaterisk@users.noreply.github.com> Date: Tue, 23 Oct 2018 16:55:33 +1100 Subject: [PATCH 254/282] Add swap_dims to relevant 'See also' sections (#2463) --- xarray/core/common.py | 1 + xarray/core/dataset.py | 5 +++++ xarray/core/groupby.py | 1 + 3 files changed, 7 insertions(+) diff --git a/xarray/core/common.py b/xarray/core/common.py index c74b1fa080b..6c03775a5dd 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -343,6 +343,7 @@ def assign_coords(self, **kwargs): See also -------- Dataset.assign + Dataset.swap_dims """ data = self.copy(deep=False) results = self._calc_assign_results(kwargs) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index c8586d1d408..4f06782918a 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1112,6 +1112,10 @@ def set_coords(self, names, inplace=False): Returns ------- Dataset + + See also + -------- + Dataset.swap_dims """ # TODO: allow inserting new coordinates with this method, like # DataFrame.set_index? @@ -2282,6 +2286,7 @@ def set_index(self, indexes=None, append=False, inplace=False, See Also -------- Dataset.reset_index + Dataset.swap_dims """ indexes = either_dict_or_kwargs(indexes, indexes_kwargs, 'set_index') variables, coord_names = merge_indexes(indexes, self._variables, diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index 3842c642047..dc23eae8b76 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -423,6 +423,7 @@ def assign_coords(self, **kwargs): See also -------- Dataset.assign_coords + Dataset.swap_dims """ return self.apply(lambda ds: ds.assign_coords(**kwargs)) From 6008dc43bbf3b161cf822142082398073aba9f9d Mon Sep 17 00:00:00 2001 From: George Geddes Date: Tue, 23 Oct 2018 01:59:46 -0400 Subject: [PATCH 255/282] Make Dataset.copy docstring match behavior (#2491) --- xarray/core/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 4f06782918a..756cd795540 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -727,7 +727,7 @@ def copy(self, deep=False, data=None): ---------- deep : bool, optional Whether each component variable is loaded into memory and copied onto - the new object. Default is True. + the new object. Default is False. data : dict-like, optional Data to use in the new object. Each item in `data` must have same shape as corresponding data variable in original. When `data` is From 5ebed791db006075701c16b2e8eb9566207f2548 Mon Sep 17 00:00:00 2001 From: Maximilian Maahn Date: Tue, 23 Oct 2018 02:19:22 -0600 Subject: [PATCH 256/282] ENH: Plotting support for interval coordinates: groupby_bins (#2152) * ENH: Plotting for groupby_bins DataArrays created with e.g. groupy_bins have coords containing of pd._libs.interval.Interval. For plotting, the pd._libs.interval.Interval is replaced with the interval's center point. '_center' is appended to teh label * changed pd._libs.interval.Interval to pd.Interval * Assign new variable with _interval_to_mid_points instead of mutating original variable. Note that this changes the the type of xplt from DataArray to np.array in the line function. * '_center' added to label only for 1d plot * added tests * missing whitespace * Simplified test * simplified tests once more * 1d plots now defaults to step plot New bool keyword `interval_step_plot` to turn it off. * non-uniform bin spacing for pcolormesh * Added step plot function * bugfix: linestyle == '' results in no line plotted * Adapted to upstream changes * Added _resolve_intervals_2dplot function, simplified code * Added documentation * typo in documentation * Fixed bug introduced by upstream change * Refactor out utility functions. * Fix test. * Add whats-new. * Remove duplicate whats new entry. :/ * Make things neater. --- doc/plotting.rst | 32 +++++++++++ doc/whats-new.rst | 3 + xarray/plot/__init__.py | 3 +- xarray/plot/plot.py | 114 +++++++++++++++++++++++++++++++------- xarray/plot/utils.py | 68 ++++++++++++++++++++++- xarray/tests/test_plot.py | 28 ++++++++++ 6 files changed, 224 insertions(+), 24 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index 37fdf03804a..6e3498c126f 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -222,9 +222,41 @@ It is also possible to make line plots such that the data are on the x-axis and @savefig plotting_example_xy_kwarg.png air.isel(time=10, lon=[10, 11]).plot(y='lat', hue='lon') +Step plots +~~~~~~~~~~ + +As an alternative, also a step plot similar to matplotlib's ``plt.step`` can be +made using 1D data. + +.. ipython:: python + + @savefig plotting_example_step.png width=4in + air1d[:20].plot.step(where='mid') + +The argument ``where`` defines where the steps should be placed, options are +``'pre'`` (default), ``'post'``, and ``'mid'``. This is particularly handy +when plotting data grouped with :py:func:`xarray.Dataset.groupby_bins`. + +.. ipython:: python + + air_grp = air.mean(['time','lon']).groupby_bins('lat',[0,23.5,66.5,90]) + air_mean = air_grp.mean() + air_std = air_grp.std() + air_mean.plot.step() + (air_mean + air_std).plot.step(ls=':') + (air_mean - air_std).plot.step(ls=':') + plt.ylim(-20,30) + @savefig plotting_example_step_groupby.png width=4in + plt.title('Zonal mean temperature') + +In this case, the actual boundaries of the bins are used and the ``where`` argument +is ignored. + + Other axes kwargs ----------------- + The keyword arguments ``xincrease`` and ``yincrease`` let you control the axes direction. .. ipython:: python diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 9a55b9380b9..6512d48d7d8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -63,6 +63,9 @@ Enhancements By `Deepak Cherian `_. - Added support for Python 3.7. (:issue:`2271`). By `Joe Hamman `_. +- Added support for plotting data with `pandas.Interval` coordinates, such as those + created by :py:meth:`~xarray.DataArray.groupby_bins` + By `Maximilian Maahn `_. - Added :py:meth:`~xarray.CFTimeIndex.shift` for shifting the values of a CFTimeIndex by a specified frequency. (:issue:`2244`). By `Spencer Clark `_. diff --git a/xarray/plot/__init__.py b/xarray/plot/__init__.py index fe2c604a89e..4b53b22243c 100644 --- a/xarray/plot/__init__.py +++ b/xarray/plot/__init__.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from .plot import (plot, line, contourf, contour, +from .plot import (plot, line, step, contourf, contour, hist, imshow, pcolormesh) from .facetgrid import FacetGrid @@ -9,6 +9,7 @@ __all__ = [ 'plot', 'line', + 'step', 'contour', 'contourf', 'hist', diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index a43cee14eb3..7129157ec7f 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -20,7 +20,9 @@ from .facetgrid import FacetGrid from .utils import ( - ROBUST_PERCENTILE, _determine_cmap_params, _infer_xy_labels, get_axis, + ROBUST_PERCENTILE, _determine_cmap_params, _infer_xy_labels, + _interval_to_double_bound_points, _interval_to_mid_points, + _resolve_intervals_2dplot, _valid_other_type, get_axis, import_matplotlib_pyplot, label_from_attrs) @@ -36,27 +38,20 @@ def _valid_numpy_subdtype(x, numpy_types): return any(np.issubdtype(x.dtype, t) for t in numpy_types) -def _valid_other_type(x, types): - """ - Do all elements of x have a type from types? - """ - return all(any(isinstance(el, t) for t in types) for el in np.ravel(x)) - - def _ensure_plottable(*args): """ Raise exception if there is anything in args that can't be plotted on an - axis. + axis by matplotlib. """ numpy_types = [np.floating, np.integer, np.timedelta64, np.datetime64] other_types = [datetime] for x in args: - if not (_valid_numpy_subdtype(np.array(x), numpy_types) or - _valid_other_type(np.array(x), other_types)): + if not (_valid_numpy_subdtype(np.array(x), numpy_types) + or _valid_other_type(np.array(x), other_types)): raise TypeError('Plotting requires coordinates to be numeric ' 'or dates of type np.datetime64 or ' - 'datetime.datetime.') + 'datetime.datetime or pd.Interval.') def _easy_facetgrid(darray, plotfunc, x, y, row=None, col=None, @@ -350,9 +345,30 @@ def line(darray, *args, **kwargs): xplt, yplt, hueplt, xlabel, ylabel, huelabel = \ _infer_line_data(darray, x, y, hue) - _ensure_plottable(xplt) + # Remove pd.Intervals if contained in xplt.values. + if _valid_other_type(xplt.values, [pd.Interval]): + # Is it a step plot? (see matplotlib.Axes.step) + if kwargs.get('linestyle', '').startswith('steps-'): + xplt_val, yplt_val = _interval_to_double_bound_points(xplt.values, + yplt.values) + # Remove steps-* to be sure that matplotlib is not confused + kwargs['linestyle'] = (kwargs['linestyle'] + .replace('steps-pre', '') + .replace('steps-post', '') + .replace('steps-mid', '')) + if kwargs['linestyle'] == '': + kwargs.pop('linestyle') + else: + xplt_val = _interval_to_mid_points(xplt.values) + yplt_val = yplt.values + xlabel += '_center' + else: + xplt_val = xplt.values + yplt_val = yplt.values - primitive = ax.plot(xplt, yplt, *args, **kwargs) + _ensure_plottable(xplt_val, yplt_val) + + primitive = ax.plot(xplt_val, yplt_val, *args, **kwargs) if _labels: if xlabel is not None: @@ -383,6 +399,46 @@ def line(darray, *args, **kwargs): return primitive +def step(darray, *args, **kwargs): + """ + Step plot of DataArray index against values + + Similar to :func:`matplotlib:matplotlib.pyplot.step` + + Parameters + ---------- + where : {'pre', 'post', 'mid'}, optional, default 'pre' + Define where the steps should be placed: + - 'pre': The y value is continued constantly to the left from + every *x* position, i.e. the interval ``(x[i-1], x[i]]`` has the + value ``y[i]``. + - 'post': The y value is continued constantly to the right from + every *x* position, i.e. the interval ``[x[i], x[i+1])`` has the + value ``y[i]``. + - 'mid': Steps occur half-way between the *x* positions. + Note that this parameter is ignored if the x coordinate consists of + :py:func:`pandas.Interval` values, e.g. as a result of + :py:func:`xarray.Dataset.groupby_bins`. In this case, the actual + boundaries of the interval are used. + + *args, **kwargs : optional + Additional arguments following :py:func:`xarray.plot.line` + + """ + if ('ls' in kwargs.keys()) and ('linestyle' not in kwargs.keys()): + kwargs['linestyle'] = kwargs.pop('ls') + + where = kwargs.pop('where', 'pre') + + if where not in ('pre', 'post', 'mid'): + raise ValueError("'where' argument to step must be " + "'pre', 'post' or 'mid'") + + kwargs['linestyle'] = 'steps-' + where + kwargs.get('linestyle', '') + + return line(darray, *args, **kwargs) + + def hist(darray, figsize=None, size=None, aspect=None, ax=None, **kwargs): """ Histogram of DataArray @@ -500,6 +556,10 @@ def hist(self, ax=None, **kwargs): def line(self, *args, **kwargs): return line(self._da, *args, **kwargs) + @functools.wraps(step) + def step(self, *args, **kwargs): + return step(self._da, *args, **kwargs) + def _rescale_imshow_rgb(darray, vmin, vmax, robust): assert robust or vmin is not None or vmax is not None @@ -740,7 +800,11 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, # Pass the data as a masked ndarray too zval = darray.to_masked_array(copy=False) - _ensure_plottable(xval, yval) + # Replace pd.Intervals if contained in xval or yval. + xplt, xlab_extra = _resolve_intervals_2dplot(xval, plotfunc.__name__) + yplt, ylab_extra = _resolve_intervals_2dplot(yval, plotfunc.__name__) + + _ensure_plottable(xplt, yplt) if 'contour' in plotfunc.__name__ and levels is None: levels = 7 # this is the matplotlib default @@ -780,7 +844,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, "in xarray") ax = get_axis(figsize, size, aspect, ax) - primitive = plotfunc(xval, yval, zval, ax=ax, cmap=cmap_params['cmap'], + primitive = plotfunc(xplt, yplt, zval, ax=ax, cmap=cmap_params['cmap'], vmin=cmap_params['vmin'], vmax=cmap_params['vmax'], norm=cmap_params['norm'], @@ -788,8 +852,8 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, # Label the plot with metadata if add_labels: - ax.set_xlabel(label_from_attrs(darray[xlab])) - ax.set_ylabel(label_from_attrs(darray[ylab])) + ax.set_xlabel(label_from_attrs(darray[xlab], xlab_extra)) + ax.set_ylabel(label_from_attrs(darray[ylab], ylab_extra)) ax.set_title(darray._title_for_slice()) if add_colorbar: @@ -818,7 +882,7 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, # Do this without calling autofmt_xdate so that x-axes ticks # on other subplots (if any) are not deleted. # https://stackoverflow.com/questions/17430105/autofmt-xdate-deletes-x-axis-labels-of-all-subplots - if np.issubdtype(xval.dtype, np.datetime64): + if np.issubdtype(xplt.dtype, np.datetime64): for xlabels in ax.get_xticklabels(): xlabels.set_rotation(30) xlabels.set_ha('right') @@ -1019,14 +1083,22 @@ def pcolormesh(x, y, z, ax, infer_intervals=None, **kwargs): else: infer_intervals = True - if infer_intervals: + if (infer_intervals and + ((np.shape(x)[0] == np.shape(z)[1]) or + ((x.ndim > 1) and (np.shape(x)[1] == np.shape(z)[1])))): if len(x.shape) == 1: x = _infer_interval_breaks(x, check_monotonic=True) - y = _infer_interval_breaks(y, check_monotonic=True) else: # we have to infer the intervals on both axes x = _infer_interval_breaks(x, axis=1) x = _infer_interval_breaks(x, axis=0) + + if (infer_intervals and + (np.shape(y)[0] == np.shape(z)[0])): + if len(y.shape) == 1: + y = _infer_interval_breaks(y, check_monotonic=True) + else: + # we have to infer the intervals on both axes y = _infer_interval_breaks(y, axis=1) y = _infer_interval_breaks(y, axis=0) diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 111fd4de6e5..41f61554739 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -1,9 +1,11 @@ from __future__ import absolute_import, division, print_function +import itertools import textwrap import warnings import numpy as np +import pandas as pd from ..core.options import OPTIONS from ..core.pycompat import basestring @@ -367,7 +369,7 @@ def get_axis(figsize, size, aspect, ax): return ax -def label_from_attrs(da): +def label_from_attrs(da, extra=''): ''' Makes informative labels if variable metadata (attrs) follows CF conventions. ''' @@ -385,4 +387,66 @@ def label_from_attrs(da): else: units = '' - return '\n'.join(textwrap.wrap(name + units, 30)) + return '\n'.join(textwrap.wrap(name + extra + units, 30)) + + +def _interval_to_mid_points(array): + """ + Helper function which returns an array + with the Intervals' mid points. + """ + + return np.array([x.mid for x in array]) + + +def _interval_to_bound_points(array): + """ + Helper function which returns an array + with the Intervals' boundaries. + """ + + array_boundaries = np.array([x.left for x in array]) + array_boundaries = np.concatenate( + (array_boundaries, np.array([array[-1].right]))) + + return array_boundaries + + +def _interval_to_double_bound_points(xarray, yarray): + """ + Helper function to deal with a xarray consisting of pd.Intervals. Each + interval is replaced with both boundaries. I.e. the length of xarray + doubles. yarray is modified so it matches the new shape of xarray. + """ + + xarray1 = np.array([x.left for x in xarray]) + xarray2 = np.array([x.right for x in xarray]) + + xarray = list(itertools.chain.from_iterable(zip(xarray1, xarray2))) + yarray = list(itertools.chain.from_iterable(zip(yarray, yarray))) + + return xarray, yarray + + +def _resolve_intervals_2dplot(val, func_name): + """ + Helper function to replace the values of a coordinate array containing + pd.Interval with their mid-points or - for pcolormesh - boundaries which + increases length by 1. + """ + label_extra = '' + if _valid_other_type(val, [pd.Interval]): + if func_name == 'pcolormesh': + val = _interval_to_bound_points(val) + else: + val = _interval_to_mid_points(val) + label_extra = '_center' + + return val, label_extra + + +def _valid_other_type(x, types): + """ + Do all elements of x have a type from types? + """ + return all(any(isinstance(el, t) for t in types) for el in np.ravel(x)) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index d2a43bc5b52..306988744d6 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -364,6 +364,10 @@ def test_convenient_facetgrid_4d(self): with raises_regex(ValueError, '[Ff]acet'): d.plot(x='x', y='y', col='columns', ax=plt.gca()) + def test_coord_with_interval(self): + bins = [-1, 0, 1, 2] + self.darray.groupby_bins('dim_0', bins).mean(xr.ALL_DIMS).plot() + class TestPlot1D(PlotTestCase): @pytest.fixture(autouse=True) @@ -438,6 +442,20 @@ def test_slice_in_title(self): assert 'd = 10' == title +class TestPlotStep(PlotTestCase): + @pytest.fixture(autouse=True) + def setUp(self): + self.darray = DataArray(easy_array((2, 3, 4))) + + def test_step(self): + self.darray[0, 0].plot.step() + + def test_coord_with_interval_step(self): + bins = [-1, 0, 1, 2] + self.darray.groupby_bins('dim_0', bins).mean(xr.ALL_DIMS).plot.step() + assert len(plt.gca().lines[0].get_xdata()) == ((len(bins) - 1) * 2) + + class TestPlotHistogram(PlotTestCase): @pytest.fixture(autouse=True) def setUp(self): @@ -473,6 +491,10 @@ def test_plot_nans(self): self.darray[0, 0, 0] = np.nan self.darray.plot.hist() + def test_hist_coord_with_interval(self): + (self.darray.groupby_bins('dim_0', [-1, 0, 1, 2]).mean(xr.ALL_DIMS) + .plot.hist(range=(-1, 2))) + @requires_matplotlib class TestDetermineCmapParams(object): @@ -1129,6 +1151,12 @@ def test_cmap_and_color_both(self): with pytest.raises(ValueError): self.plotmethod(colors='k', cmap='RdBu') + def test_2d_coord_with_interval(self): + for dim in self.darray.dims: + gp = self.darray.groupby_bins(dim, range(15)).mean(dim) + for kind in ['imshow', 'pcolormesh', 'contourf', 'contour']: + getattr(gp.plot, kind)() + def test_colormap_error_norm_and_vmin_vmax(self): norm = mpl.colors.LogNorm(0.1, 1e1) From b8c4b7862c87223f32e13d7c454a8f275998f828 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Tue, 23 Oct 2018 20:28:47 +0200 Subject: [PATCH 257/282] Fix tests to new coordinate names in cfgrib>=0.9.2. (#2502) --- ci/requirements-py36.yml | 2 +- xarray/tests/test_backends.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index fc272984237..321f3087ea2 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -27,4 +27,4 @@ dependencies: - pytest-cov - pydap - lxml - - cfgrib + - cfgrib>=0.9.2 diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a274b41c424..c6a2df733fa 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2467,7 +2467,7 @@ def test_weakrefs(self): class TestCfGrib(object): def test_read(self): - expected = {'number': 2, 'time': 3, 'air_pressure': 2, 'latitude': 3, + expected = {'number': 2, 'time': 3, 'isobaricInhPa': 2, 'latitude': 3, 'longitude': 4} with open_example_dataset('example.grib', engine='cfgrib') as ds: assert ds.dims == expected @@ -2476,7 +2476,7 @@ def test_read(self): def test_read_filter_by_keys(self): kwargs = {'filter_by_keys': {'shortName': 't'}} - expected = {'number': 2, 'time': 3, 'air_pressure': 2, 'latitude': 3, + expected = {'number': 2, 'time': 3, 'isobaricInhPa': 2, 'latitude': 3, 'longitude': 4} with open_example_dataset('example.grib', engine='cfgrib', backend_kwargs=kwargs) as ds: From d77db21071e6c2ec0267096d5313fcd349623446 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Thu, 25 Oct 2018 11:26:58 -0400 Subject: [PATCH 258/282] Iterate over data_vars only (#2506) * iterate over data_vars * whats new --- doc/whats-new.rst | 3 +++ xarray/core/dataset.py | 25 +++---------------------- xarray/tests/test_dataset.py | 12 ++++-------- 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 6512d48d7d8..af7a7982df5 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,6 +33,9 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ +- Iterating over a ``Dataset`` now includes only data variables, not coordinates. + Similarily, calling ``len`` and ``bool`` on a ``Dataset`` now + includes only data variables - Xarray's storage backends now automatically open and close files when necessary, rather than requiring opening a file with ``autoclose=True``. A global least-recently-used cache is used to store open files; the default diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 756cd795540..534029b9aff 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -930,32 +930,13 @@ def __contains__(self, key): return key in self._variables def __len__(self): - warnings.warn('calling len() on an xarray.Dataset will change in ' - 'xarray v0.11 to only include data variables, not ' - 'coordinates. Call len() on the Dataset.variables ' - 'property instead, like ``len(ds.variables)``, to ' - 'preserve existing behavior in a forwards compatible ' - 'manner.', - FutureWarning, stacklevel=2) - return len(self._variables) + return len(self.data_vars) def __bool__(self): - warnings.warn('casting an xarray.Dataset to a boolean will change in ' - 'xarray v0.11 to only include data variables, not ' - 'coordinates. Cast the Dataset.variables property ' - 'instead to preserve existing behavior in a forwards ' - 'compatible manner.', - FutureWarning, stacklevel=2) - return bool(self._variables) + return bool(self.data_vars) def __iter__(self): - warnings.warn('iteration over an xarray.Dataset will change in xarray ' - 'v0.11 to only include data variables, not coordinates. ' - 'Iterate over the Dataset.variables property instead to ' - 'preserve existing behavior in a forwards compatible ' - 'manner.', - FutureWarning, stacklevel=2) - return iter(self._variables) + return iter(self.data_vars) def __array__(self, dtype=None): raise TypeError('cannot directly convert an xarray.Dataset into a ' diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 89704653e92..adc809662f8 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -420,16 +420,12 @@ def test_properties(self): assert isinstance(ds.dims.mapping, utils.SortedKeysDict) assert type(ds.dims.mapping.mapping) is dict # noqa - with pytest.warns(FutureWarning): - assert list(ds) == list(ds.variables) - with pytest.warns(FutureWarning): - assert list(ds.keys()) == list(ds.variables) + assert list(ds) == list(ds.data_vars) + assert list(ds.keys()) == list(ds.data_vars) assert 'aasldfjalskdfj' not in ds.variables assert 'dim1' in repr(ds.variables) - with pytest.warns(FutureWarning): - assert len(ds) == 7 - with pytest.warns(FutureWarning): - assert bool(ds) + assert len(ds) == 3 + assert bool(ds) assert list(ds.data_vars) == ['var1', 'var2', 'var3'] assert list(ds.data_vars.keys()) == ['var1', 'var2', 'var3'] From b2a377f8dd215a0e1d3cbab61e6ae930347f4063 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 25 Oct 2018 09:06:52 -0700 Subject: [PATCH 259/282] facetgrid: properly support cbar_kwargs. (#2444) * facetgrid: properly support cbar_kwargs. * Update doc/plotting.rst * Update xarray/plot/facetgrid.py * Update whats-new.rst --- doc/plotting.rst | 3 ++- doc/whats-new.rst | 3 +++ xarray/plot/facetgrid.py | 7 ++++++- xarray/tests/test_plot.py | 17 +++++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index 6e3498c126f..95e63cbff05 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -527,7 +527,8 @@ Faceted plotting supports other arguments common to xarray 2d plots. @savefig plot_facet_robust.png g = hasoutliers.plot.pcolormesh('lon', 'lat', col='time', col_wrap=3, - robust=True, cmap='viridis') + robust=True, cmap='viridis', + cbar_kwargs={'label': 'this has outliers'}) FacetGrid Objects ~~~~~~~~~~~~~~~~~ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index af7a7982df5..4ae498c3bb7 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -86,6 +86,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- ``FacetGrid`` now properly uses the ``cbar_kwargs`` keyword argument. + (:issue:`1504`, :issue:`1717`) + By `Deepak Cherian `_. - Addition and subtraction operators used with a CFTimeIndex now preserve the index's type. (:issue:`2244`). By `Spencer Clark `_. diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index 32a954a3fcd..f133e7806a3 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -223,6 +223,11 @@ def map_dataarray(self, func, x, y, **kwargs): cmapkw = kwargs.get('cmap') colorskw = kwargs.get('colors') + cbar_kwargs = kwargs.pop('cbar_kwargs', {}) + cbar_kwargs = {} if cbar_kwargs is None else dict(cbar_kwargs) + + if kwargs.get('cbar_ax', None) is not None: + raise ValueError('cbar_ax not supported by FacetGrid.') # colors is mutually exclusive with cmap if cmapkw and colorskw: @@ -264,7 +269,7 @@ def map_dataarray(self, func, x, y, **kwargs): self._finalize_grid(x, y) if kwargs.get('add_colorbar', True): - self.add_colorbar() + self.add_colorbar(**cbar_kwargs) return self diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 306988744d6..38cf68e47cc 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1147,6 +1147,23 @@ def test_facetgrid_cmap(self): # check that all colormaps are the same assert len(set(m.get_cmap().name for m in fg._mappables)) == 1 + def test_facetgrid_cbar_kwargs(self): + a = easy_array((10, 15, 2, 3)) + d = DataArray(a, dims=['y', 'x', 'columns', 'rows']) + g = self.plotfunc(d, x='x', y='y', col='columns', row='rows', + cbar_kwargs={'label': 'test_label'}) + + # catch contour case + if hasattr(g, 'cbar'): + assert g.cbar._label == 'test_label' + + def test_facetgrid_no_cbar_ax(self): + a = easy_array((10, 15, 2, 3)) + d = DataArray(a, dims=['y', 'x', 'columns', 'rows']) + with pytest.raises(ValueError): + g = self.plotfunc(d, x='x', y='y', col='columns', row='rows', + cbar_ax=1) + def test_cmap_and_color_both(self): with pytest.raises(ValueError): self.plotmethod(colors='k', cmap='RdBu') From 2a4691321a3c13baf0a5615f16740435621b153d Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 25 Oct 2018 14:23:35 -0700 Subject: [PATCH 260/282] Add satpy to related projects --- doc/related-projects.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/related-projects.rst b/doc/related-projects.rst index 524ea3b9d8d..cf89c715bc7 100644 --- a/doc/related-projects.rst +++ b/doc/related-projects.rst @@ -24,6 +24,7 @@ Geosciences subclass. - `Regionmask `_: plotting and creation of masks of spatial regions - `salem `_: Adds geolocalised subsetting, masking, and plotting operations to xarray's data structures via accessors. +- `SatPy `_ : Library for reading and manipulating meteorological remote sensing data and writing it to various image and data file formats. - `Spyfit `_: FTIR spectroscopy of the atmosphere - `windspharm `_: Spherical harmonic wind analysis in Python. From 5940100761478604080523ebb1291ecff90e779e Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 25 Oct 2018 19:04:32 -0700 Subject: [PATCH 261/282] Remove .T as shortcut for transpose() (#2509) * Remove .T as shortcut for transpose() * fix whats-new * remove Dataset.__dir__ * Update whats-new.rst * Update whats-new.rst --- doc/whats-new.rst | 2 ++ xarray/core/dataset.py | 14 -------------- xarray/tests/test_dataset.py | 4 ---- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4ae498c3bb7..e1744e28077 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,6 +33,8 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ +- ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. + Call :py:meth:`Dataset.transpose` directly instead. - Iterating over a ``Dataset`` now includes only data variables, not coordinates. Similarily, calling ``len`` and ``bool`` on a ``Dataset`` now includes only data variables diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 534029b9aff..983270cf425 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -916,13 +916,6 @@ def _item_sources(self): return [self.data_vars, self.coords, {d: self[d] for d in self.dims}, LevelCoordinatesSource(self)] - def __dir__(self): - # In order to suppress a deprecation warning in Ipython autocompletion - # .T is explicitly removed from __dir__. GH: issue 1675 - d = super(Dataset, self).__dir__() - d.remove('T') - return d - def __contains__(self, key): """The 'in' operator will return true or false depending on whether 'key' is an array in the dataset or not. @@ -2647,13 +2640,6 @@ def transpose(self, *dims): ds._variables[name] = var.transpose(*var_dims) return ds - @property - def T(self): - warnings.warn('xarray.Dataset.T has been deprecated as an alias for ' - '`.transpose()`. It will be removed in xarray v0.11.', - FutureWarning, stacklevel=2) - return self.transpose() - def dropna(self, dim, how='any', thresh=None, subset=None): """Returns a new dataset with dropped labels for missing values along the provided dimension. diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index adc809662f8..aa226ff1ce8 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -3801,10 +3801,6 @@ def test_dataset_transpose(self): expected = ds.apply(lambda x: x.transpose()) assert_identical(expected, actual) - with pytest.warns(FutureWarning): - actual = ds.T - assert_identical(expected, actual) - actual = ds.transpose('x', 'y') expected = ds.apply(lambda x: x.transpose('x', 'y')) assert_identical(expected, actual) From b622c5e7da928524ef949d9e389f6c7f38644494 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 26 Oct 2018 10:50:35 -0400 Subject: [PATCH 262/282] Remove Dataset.T from api-hidden.rst (#2515) --- doc/api-hidden.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/api-hidden.rst b/doc/api-hidden.rst index 0e8143c72ea..0d8189d21cf 100644 --- a/doc/api-hidden.rst +++ b/doc/api-hidden.rst @@ -39,7 +39,6 @@ Dataset.imag Dataset.round Dataset.real - Dataset.T Dataset.cumsum Dataset.cumprod Dataset.rank From 2f0096cfab62523f26232bedf3debaba5f58d337 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sat, 27 Oct 2018 09:34:52 -0700 Subject: [PATCH 263/282] Make sure datetime object arrays are converted to datetime64 (#2513) Fixes #2512 --- doc/whats-new.rst | 3 +++ xarray/core/variable.py | 2 +- xarray/tests/test_variable.py | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e1744e28077..4497c57e5f2 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -94,6 +94,9 @@ Bug fixes - Addition and subtraction operators used with a CFTimeIndex now preserve the index's type. (:issue:`2244`). By `Spencer Clark `_. +- We now properly handle arrays of ``datetime.datetime`` and ``datetime.timedelta`` + provided as coordinates. (:issue:`2512`) + By `Deepak Cherian `_. diff --git a/xarray/core/variable.py b/xarray/core/variable.py index c003d52aab2..fefd48b449c 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -95,7 +95,7 @@ def as_variable(obj, name=None): 'cannot set variable %r with %r-dimensional data ' 'without explicit dimension names. Pass a tuple of ' '(dims, data) instead.' % (name, data.ndim)) - obj = Variable(name, obj, fastpath=True) + obj = Variable(name, data, fastpath=True) else: raise TypeError('unable to convert object into a variable without an ' 'explicit list of dimensions: %r' % obj) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 52289a15d72..3753221f352 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -992,6 +992,13 @@ def test_as_variable(self): ValueError, 'has more than 1-dimension'): as_variable(expected, name='x') + # test datetime, timedelta conversion + dt = np.array([datetime(1999, 1, 1) + timedelta(days=x) + for x in range(10)]) + assert as_variable(dt, 'time').dtype.kind == 'M' + td = np.array([timedelta(days=x) for x in range(10)]) + assert as_variable(td, 'time').dtype.kind == 'm' + def test_repr(self): v = Variable(['time', 'x'], [[1, 2, 3], [4, 5, 6]], {'foo': 'bar'}) expected = dedent(""" From c2a6902f090e063692c53e1dacd6c20e584d8e80 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sat, 27 Oct 2018 20:38:59 -0400 Subject: [PATCH 264/282] Fix bug where OverflowError is not being raised (#2519) --- doc/whats-new.rst | 4 ++++ xarray/coding/times.py | 7 ++++++- xarray/tests/test_coding_times.py | 16 +++++++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4497c57e5f2..918106a45df 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -123,6 +123,10 @@ Bug fixes By `Spencer Clark `_. - Avoid use of Dask's deprecated ``get=`` parameter in tests by `Matthew Rocklin `_. +- An ``OverflowError`` is now accurately raised and caught during the + encoding process if a reference date is used that is so distant that + the dates must be encoded using cftime rather than NumPy (:issue:`2272`). + By `Spencer Clark `_. .. _whats-new.0.10.9: diff --git a/xarray/coding/times.py b/xarray/coding/times.py index dff7e75bdcf..16380976def 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -361,7 +361,12 @@ def encode_cf_datetime(dates, units=None, calendar=None): delta_units = _netcdf_to_numpy_timeunit(delta) time_delta = np.timedelta64(1, delta_units).astype('timedelta64[ns]') ref_date = np.datetime64(pd.Timestamp(ref_date)) - num = (dates - ref_date) / time_delta + + # Wrap the dates in a DatetimeIndex to do the subtraction to ensure + # an OverflowError is raised if the ref_date is too far away from + # dates to be encoded (GH 2272). + num = (pd.DatetimeIndex(dates.ravel()) - ref_date) / time_delta + num = num.values.reshape(dates.shape) except (OutOfBoundsDatetime, OverflowError): num = _encode_datetime_with_cftime(dates, units, calendar) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 10a1a956b27..8e47bd37eac 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -8,7 +8,8 @@ import pytest from xarray import DataArray, Variable, coding, decode_cf, set_options -from xarray.coding.times import _import_cftime +from xarray.coding.times import (_import_cftime, decode_cf_datetime, + encode_cf_datetime) from xarray.coding.variables import SerializationWarning from xarray.core.common import contains_cftime_datetimes @@ -763,3 +764,16 @@ def test_contains_cftime_datetimes_non_cftimes(non_cftime_data): @pytest.mark.parametrize('non_cftime_data', [DataArray([]), DataArray([1, 2])]) def test_contains_cftime_datetimes_non_cftimes_dask(non_cftime_data): assert not contains_cftime_datetimes(non_cftime_data.chunk()) + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize('shape', [(24,), (8, 3), (2, 4, 3)]) +def test_encode_datetime_overflow(shape): + # Test for fix to GH 2272 + dates = pd.date_range('2100', periods=24).values.reshape(shape) + units = 'days since 1800-01-01' + calendar = 'standard' + + num, _, _ = encode_cf_datetime(dates, units, calendar) + roundtrip = decode_cf_datetime(num, units, calendar) + np.testing.assert_array_equal(dates, roundtrip) From a2583334f5c40f0c8023146dcf0a82ca65875f32 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 28 Oct 2018 10:56:17 -0700 Subject: [PATCH 265/282] Finish deprecation cycle for DataArray.__contains__ checking array values (#2520) Fixes GH1267 --- doc/whats-new.rst | 13 ++++++++----- xarray/core/dataarray.py | 6 +----- xarray/tests/test_dataarray.py | 7 ++++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 918106a45df..3a644ddf63a 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,11 +33,14 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ -- ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. - Call :py:meth:`Dataset.transpose` directly instead. -- Iterating over a ``Dataset`` now includes only data variables, not coordinates. - Similarily, calling ``len`` and ``bool`` on a ``Dataset`` now - includes only data variables +- Finished deprecation cycles: + - ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. + Call :py:meth:`Dataset.transpose` directly instead. + - Iterating over a ``Dataset`` now includes only data variables, not coordinates. + Similarily, calling ``len`` and ``bool`` on a ``Dataset`` now + includes only data variables. + - ``DataArray.__contains__`` (used by Python's ``in`` operator) now checks + array data, not coordinates. - Xarray's storage backends now automatically open and close files when necessary, rather than requiring opening a file with ``autoclose=True``. A global least-recently-used cache is used to store open files; the default diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index f131b003a69..7dc867c98ed 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -503,11 +503,7 @@ def _item_sources(self): LevelCoordinatesSource(self)] def __contains__(self, key): - warnings.warn( - 'xarray.DataArray.__contains__ currently checks membership in ' - 'DataArray.coords, but in xarray v0.11 will change to check ' - 'membership in array values.', FutureWarning, stacklevel=2) - return key in self._coords + return key in self.data @property def loc(self): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index e49b6cdf517..433a669e340 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -618,9 +618,9 @@ def get_data(): da[dict(x=ind)] = value # should not raise def test_contains(self): - data_array = DataArray(1, coords={'x': 2}) - with pytest.warns(FutureWarning): - assert 'x' in data_array + data_array = DataArray([1, 2]) + assert 1 in data_array + assert 3 not in data_array def test_attr_sources_multiindex(self): # make sure attr-style access for multi-index levels @@ -2533,6 +2533,7 @@ def test_upsample_interpolate_regression_1605(self): assert_allclose(actual, expected, rtol=1e-16) @requires_dask + @requires_scipy def test_upsample_interpolate_dask(self): import dask.array as da From 3176d8a241ff2bcfaa93536a59497c637358b022 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 29 Oct 2018 21:00:43 -0400 Subject: [PATCH 266/282] Remove tests where answers change in cftime 1.0.2.1 (#2522) --- xarray/tests/test_coding_times.py | 38 ++++++++++++++----------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 8e47bd37eac..f76b8c3ceab 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -576,28 +576,24 @@ def test_infer_datetime_units(dates, expected): assert expected == coding.times.infer_datetime_units(dates) +_CFTIME_DATETIME_UNITS_TESTS = [ + ([(1900, 1, 1), (1900, 1, 1)], 'days since 1900-01-01 00:00:00.000000'), + ([(1900, 1, 1), (1900, 1, 2), (1900, 1, 2, 0, 0, 1)], + 'seconds since 1900-01-01 00:00:00.000000'), + ([(1900, 1, 1), (1900, 1, 8), (1900, 1, 16)], + 'days since 1900-01-01 00:00:00.000000') +] + + @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -def test_infer_cftime_datetime_units(): - date_types = _all_cftime_date_types() - for date_type in date_types.values(): - for dates, expected in [ - ([date_type(1900, 1, 1), - date_type(1900, 1, 2)], - 'days since 1900-01-01 00:00:00.000000'), - ([date_type(1900, 1, 1, 12), - date_type(1900, 1, 1, 13)], - 'seconds since 1900-01-01 12:00:00.000000'), - ([date_type(1900, 1, 1), - date_type(1900, 1, 2), - date_type(1900, 1, 2, 0, 0, 1)], - 'seconds since 1900-01-01 00:00:00.000000'), - ([date_type(1900, 1, 1), - date_type(1900, 1, 2, 0, 0, 0, 5)], - 'days since 1900-01-01 00:00:00.000000'), - ([date_type(1900, 1, 1), date_type(1900, 1, 8), - date_type(1900, 1, 16)], - 'days since 1900-01-01 00:00:00.000000')]: - assert expected == coding.times.infer_datetime_units(dates) +@pytest.mark.parametrize( + 'calendar', _NON_STANDARD_CALENDARS + ['gregorian', 'proleptic_gregorian']) +@pytest.mark.parametrize(('date_args', 'expected'), + _CFTIME_DATETIME_UNITS_TESTS) +def test_infer_cftime_datetime_units(calendar, date_args, expected): + date_type = _all_cftime_date_types()[calendar] + dates = [date_type(*args) for args in date_args] + assert expected == coding.times.infer_datetime_units(dates) @pytest.mark.parametrize( From 6d55f99905d664ef73cb708cfe8c52c2c651e8dc Mon Sep 17 00:00:00 2001 From: Tom Nicholas <35968931+TomNicholas@users.noreply.github.com> Date: Tue, 30 Oct 2018 01:01:07 +0000 Subject: [PATCH 267/282] Global option to always keep/discard attrs on operations (#2482) * Added a global option to always keep or discard attrs. * Updated docs and options docstring to describe new keep_attrs global option * Updated all default keep_attrs arguments to check global option * New test to check attributes are retained properly * Implemented shoyer's suggestion so attribute permanence test now passes for reduce methods * Added tests to explicitly check that attrs are propagated correctly * Updated what's new with global keep_attrs option * Bugfix to stop failing tests in test_dataset * Test class now inherits from object for python2 compatibility * Fixes to documentation * Removed some unneccessary checks of the global keep_attrs option * Removed whitespace typo I just created * Removed some more unneccessary checks of global keep_attrs option (pointed out by dcherian) --- doc/faq.rst | 3 +- doc/whats-new.rst | 7 ++- xarray/core/common.py | 22 ++++--- xarray/core/dataarray.py | 10 +-- xarray/core/dataset.py | 25 +++++--- xarray/core/groupby.py | 35 +++++++---- xarray/core/options.py | 22 ++++++- xarray/core/resample.py | 2 +- xarray/core/variable.py | 7 ++- xarray/tests/test_options.py | 118 ++++++++++++++++++++++++++++++++++- 10 files changed, 211 insertions(+), 40 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 9313481f50a..44bc021024b 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -119,7 +119,8 @@ conventions`_. (An exception is serialization to and from netCDF files.) An implication of this choice is that we do not propagate ``attrs`` through most operations unless explicitly flagged (some methods have a ``keep_attrs`` -option). Similarly, xarray does not check for conflicts between ``attrs`` when +option, and there is a global flag for setting this to be always True or +False). Similarly, xarray does not check for conflicts between ``attrs`` when combining arrays and datasets, unless explicitly requested with the option ``compat='identical'``. The guiding principle is that metadata should not be allowed to get in the way. diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3a644ddf63a..2ffbc60622d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -82,7 +82,12 @@ Enhancements :py:meth:`~xarray.Dataset.differentiate`, :py:meth:`~xarray.DataArray.interp`, and :py:meth:`~xarray.Dataset.interp`. - By `Spencer Clark `_. + By `Spencer Clark `_ +- There is now a global option to either always keep or always discard + dataset and dataarray attrs upon operations. The option is set with + ``xarray.set_options(keep_attrs=True)``, and the default is to use the old + behaviour. + By `Tom Nicholas `_. - Added a new backend for the GRIB file format based on ECMWF *cfgrib* python driver and *ecCodes* C-library. (:issue:`2475`) By `Alessandro Amici `_, diff --git a/xarray/core/common.py b/xarray/core/common.py index 6c03775a5dd..e303c485523 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -11,6 +11,7 @@ from .arithmetic import SupportsArithmetic from .pycompat import OrderedDict, basestring, dask_array_type, suppress from .utils import Frozen, ReprObject, SortedKeysDict, either_dict_or_kwargs +from .options import _get_keep_attrs # Used as a sentinel value to indicate a all dimensions ALL_DIMS = ReprObject('') @@ -21,13 +22,13 @@ class ImplementsArrayReduce(object): def _reduce_method(cls, func, include_skipna, numeric_only): if include_skipna: def wrapped_func(self, dim=None, axis=None, skipna=None, - keep_attrs=False, **kwargs): - return self.reduce(func, dim, axis, keep_attrs=keep_attrs, + **kwargs): + return self.reduce(func, dim, axis, skipna=skipna, allow_lazy=True, **kwargs) else: - def wrapped_func(self, dim=None, axis=None, keep_attrs=False, + def wrapped_func(self, dim=None, axis=None, **kwargs): - return self.reduce(func, dim, axis, keep_attrs=keep_attrs, + return self.reduce(func, dim, axis, allow_lazy=True, **kwargs) return wrapped_func @@ -51,14 +52,14 @@ class ImplementsDatasetReduce(object): @classmethod def _reduce_method(cls, func, include_skipna, numeric_only): if include_skipna: - def wrapped_func(self, dim=None, keep_attrs=False, skipna=None, + def wrapped_func(self, dim=None, skipna=None, **kwargs): - return self.reduce(func, dim, keep_attrs, skipna=skipna, + return self.reduce(func, dim, skipna=skipna, numeric_only=numeric_only, allow_lazy=True, **kwargs) else: - def wrapped_func(self, dim=None, keep_attrs=False, **kwargs): - return self.reduce(func, dim, keep_attrs, + def wrapped_func(self, dim=None, **kwargs): + return self.reduce(func, dim, numeric_only=numeric_only, allow_lazy=True, **kwargs) return wrapped_func @@ -591,7 +592,7 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): center=center) def resample(self, freq=None, dim=None, how=None, skipna=None, - closed=None, label=None, base=0, keep_attrs=False, **indexer): + closed=None, label=None, base=0, keep_attrs=None, **indexer): """Returns a Resample object for performing resampling operations. Handles both downsampling and upsampling. If any intervals contain no @@ -659,6 +660,9 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, from .dataarray import DataArray from .resample import RESAMPLE_DIM + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) + if dim is not None: if how is None: how = 'mean' diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 7dc867c98ed..bccfd8b79d4 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -16,7 +16,7 @@ assert_coordinate_consistent, remap_label_indexers) from .dataset import Dataset, merge_indexes, split_indexes from .formatting import format_item -from .options import OPTIONS +from .options import OPTIONS, _get_keep_attrs from .pycompat import OrderedDict, basestring, iteritems, range, zip from .utils import ( decode_numpy_dict_values, either_dict_or_kwargs, ensure_us_time_resolution) @@ -1555,7 +1555,7 @@ def combine_first(self, other): """ return ops.fillna(self, other, join="outer") - def reduce(self, func, dim=None, axis=None, keep_attrs=False, **kwargs): + def reduce(self, func, dim=None, axis=None, keep_attrs=None, **kwargs): """Reduce this array by applying `func` along some dimension(s). Parameters @@ -1584,6 +1584,7 @@ def reduce(self, func, dim=None, axis=None, keep_attrs=False, **kwargs): DataArray with this object's array replaced with an array with summarized data and the indicated dimension(s) removed. """ + var = self.variable.reduce(func, dim, axis, keep_attrs, **kwargs) return self._replace_maybe_drop_dims(var) @@ -2266,7 +2267,7 @@ def sortby(self, variables, ascending=True): ds = self._to_temp_dataset().sortby(variables, ascending=ascending) return self._from_temp_dataset(ds) - def quantile(self, q, dim=None, interpolation='linear', keep_attrs=False): + def quantile(self, q, dim=None, interpolation='linear', keep_attrs=None): """Compute the qth quantile of the data along the specified dimension. Returns the qth quantiles(s) of the array elements. @@ -2312,7 +2313,7 @@ def quantile(self, q, dim=None, interpolation='linear', keep_attrs=False): q, dim=dim, keep_attrs=keep_attrs, interpolation=interpolation) return self._from_temp_dataset(ds) - def rank(self, dim, pct=False, keep_attrs=False): + def rank(self, dim, pct=False, keep_attrs=None): """Ranks the data. Equal values are assigned a rank that is the average of the ranks that @@ -2348,6 +2349,7 @@ def rank(self, dim, pct=False, keep_attrs=False): array([ 1., 2., 3.]) Dimensions without coordinates: x """ + ds = self._to_temp_dataset().rank(dim, pct=pct, keep_attrs=keep_attrs) return self._from_temp_dataset(ds) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 983270cf425..7bd99968ebb 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -28,7 +28,7 @@ from .merge import ( dataset_merge_method, dataset_update_method, merge_data_and_coords, merge_variables) -from .options import OPTIONS +from .options import OPTIONS, _get_keep_attrs from .pycompat import ( OrderedDict, basestring, dask_array_type, integer_types, iteritems, range) from .utils import ( @@ -2842,7 +2842,7 @@ def combine_first(self, other): out = ops.fillna(self, other, join="outer", dataset_join="outer") return out - def reduce(self, func, dim=None, keep_attrs=False, numeric_only=False, + def reduce(self, func, dim=None, keep_attrs=None, numeric_only=False, allow_lazy=False, **kwargs): """Reduce this dataset by applying `func` along some dimension(s). @@ -2884,6 +2884,9 @@ def reduce(self, func, dim=None, keep_attrs=False, numeric_only=False, raise ValueError('Dataset does not contain the dimensions: %s' % missing_dimensions) + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) + variables = OrderedDict() for name, var in iteritems(self._variables): reduce_dims = [dim for dim in var.dims if dim in dims] @@ -2912,7 +2915,7 @@ def reduce(self, func, dim=None, keep_attrs=False, numeric_only=False, attrs = self.attrs if keep_attrs else None return self._replace_vars_and_dims(variables, coord_names, attrs=attrs) - def apply(self, func, keep_attrs=False, args=(), **kwargs): + def apply(self, func, keep_attrs=None, args=(), **kwargs): """Apply a function over the data variables in this dataset. Parameters @@ -2957,6 +2960,8 @@ def apply(self, func, keep_attrs=False, args=(), **kwargs): variables = OrderedDict( (k, maybe_wrap_array(v, func(v, *args, **kwargs))) for k, v in iteritems(self.data_vars)) + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) attrs = self.attrs if keep_attrs else None return type(self)(variables, attrs=attrs) @@ -3621,7 +3626,7 @@ def sortby(self, variables, ascending=True): return aligned_self.isel(**indices) def quantile(self, q, dim=None, interpolation='linear', - numeric_only=False, keep_attrs=False): + numeric_only=False, keep_attrs=None): """Compute the qth quantile of the data along the specified dimension. Returns the qth quantiles(s) of the array elements for each variable @@ -3699,6 +3704,8 @@ def quantile(self, q, dim=None, interpolation='linear', # construct the new dataset coord_names = set(k for k in self.coords if k in variables) + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) attrs = self.attrs if keep_attrs else None new = self._replace_vars_and_dims(variables, coord_names, attrs=attrs) if 'quantile' in new.dims: @@ -3707,7 +3714,7 @@ def quantile(self, q, dim=None, interpolation='linear', new.coords['quantile'] = q return new - def rank(self, dim, pct=False, keep_attrs=False): + def rank(self, dim, pct=False, keep_attrs=None): """Ranks the data. Equal values are assigned a rank that is the average of the ranks that @@ -3747,6 +3754,8 @@ def rank(self, dim, pct=False, keep_attrs=False): variables[name] = var coord_names = set(self.coords) + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) attrs = self.attrs if keep_attrs else None return self._replace_vars_and_dims(variables, coord_names, attrs=attrs) @@ -3810,11 +3819,13 @@ def differentiate(self, coord, edge_order=1, datetime_unit=None): @property def real(self): - return self._unary_op(lambda x: x.real, keep_attrs=True)(self) + return self._unary_op(lambda x: x.real, + keep_attrs=True)(self) @property def imag(self): - return self._unary_op(lambda x: x.imag, keep_attrs=True)(self) + return self._unary_op(lambda x: x.imag, + keep_attrs=True)(self) def filter_by_attrs(self, **kwargs): """Returns a ``Dataset`` with variables that match specific conditions. diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index dc23eae8b76..defe72ab3ee 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -13,6 +13,7 @@ from .pycompat import integer_types, range, zip from .utils import hashable, maybe_wrap_array, peek_at, safe_cast_to_index from .variable import IndexVariable, Variable, as_variable +from .options import _get_keep_attrs def unique_value_groups(ar, sort=True): @@ -404,15 +405,17 @@ def _first_or_last(self, op, skipna, keep_attrs): # NB. this is currently only used for reductions along an existing # dimension return self._obj + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=True) return self.reduce(op, self._group_dim, skipna=skipna, keep_attrs=keep_attrs, allow_lazy=True) - def first(self, skipna=None, keep_attrs=True): + def first(self, skipna=None, keep_attrs=None): """Return the first element of each group along the group dimension """ return self._first_or_last(duck_array_ops.first, skipna, keep_attrs) - def last(self, skipna=None, keep_attrs=True): + def last(self, skipna=None, keep_attrs=None): """Return the last element of each group along the group dimension """ return self._first_or_last(duck_array_ops.last, skipna, keep_attrs) @@ -539,8 +542,8 @@ def _combine(self, applied, shortcut=False): combined = self._maybe_unstack(combined) return combined - def reduce(self, func, dim=None, axis=None, keep_attrs=False, - shortcut=True, **kwargs): + def reduce(self, func, dim=None, axis=None, + keep_attrs=None, shortcut=True, **kwargs): """Reduce the items in this group by applying `func` along some dimension(s). @@ -580,6 +583,9 @@ def reduce(self, func, dim=None, axis=None, keep_attrs=False, "warning, pass dim=xarray.ALL_DIMS explicitly.", FutureWarning, stacklevel=2) + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) + def reduce_array(ar): return ar.reduce(func, dim, axis, keep_attrs=keep_attrs, **kwargs) return self.apply(reduce_array, shortcut=shortcut) @@ -590,12 +596,12 @@ def reduce_array(ar): def _reduce_method(cls, func, include_skipna, numeric_only): if include_skipna: def wrapped_func(self, dim=DEFAULT_DIMS, axis=None, skipna=None, - keep_attrs=False, **kwargs): + keep_attrs=None, **kwargs): return self.reduce(func, dim, axis, keep_attrs=keep_attrs, skipna=skipna, allow_lazy=True, **kwargs) else: def wrapped_func(self, dim=DEFAULT_DIMS, axis=None, - keep_attrs=False, **kwargs): + keep_attrs=None, **kwargs): return self.reduce(func, dim, axis, keep_attrs=keep_attrs, allow_lazy=True, **kwargs) return wrapped_func @@ -651,7 +657,7 @@ def _combine(self, applied): combined = self._maybe_unstack(combined) return combined - def reduce(self, func, dim=None, keep_attrs=False, **kwargs): + def reduce(self, func, dim=None, keep_attrs=None, **kwargs): """Reduce the items in this group by applying `func` along some dimension(s). @@ -692,6 +698,9 @@ def reduce(self, func, dim=None, keep_attrs=False, **kwargs): elif dim is None: dim = self._group_dim + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) + def reduce_dataset(ds): return ds.reduce(func, dim, keep_attrs, **kwargs) return self.apply(reduce_dataset) @@ -701,15 +710,15 @@ def reduce_dataset(ds): @classmethod def _reduce_method(cls, func, include_skipna, numeric_only): if include_skipna: - def wrapped_func(self, dim=DEFAULT_DIMS, keep_attrs=False, + def wrapped_func(self, dim=DEFAULT_DIMS, skipna=None, **kwargs): - return self.reduce(func, dim, keep_attrs, skipna=skipna, - numeric_only=numeric_only, allow_lazy=True, - **kwargs) + return self.reduce(func, dim, + skipna=skipna, numeric_only=numeric_only, + allow_lazy=True, **kwargs) else: - def wrapped_func(self, dim=DEFAULT_DIMS, keep_attrs=False, + def wrapped_func(self, dim=DEFAULT_DIMS, **kwargs): - return self.reduce(func, dim, keep_attrs, + return self.reduce(func, dim, numeric_only=numeric_only, allow_lazy=True, **kwargs) return wrapped_func diff --git a/xarray/core/options.py b/xarray/core/options.py index 04ea0be7172..eb3013d5233 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -6,6 +6,8 @@ FILE_CACHE_MAXSIZE = 'file_cache_maxsize' CMAP_SEQUENTIAL = 'cmap_sequential' CMAP_DIVERGENT = 'cmap_divergent' +KEEP_ATTRS = 'keep_attrs' + OPTIONS = { DISPLAY_WIDTH: 80, @@ -14,6 +16,7 @@ FILE_CACHE_MAXSIZE: 128, CMAP_SEQUENTIAL: 'viridis', CMAP_DIVERGENT: 'RdBu_r', + KEEP_ATTRS: 'default' } _JOIN_OPTIONS = frozenset(['inner', 'outer', 'left', 'right', 'exact']) @@ -28,6 +31,7 @@ def _positive_integer(value): ARITHMETIC_JOIN: _JOIN_OPTIONS.__contains__, ENABLE_CFTIMEINDEX: lambda value: isinstance(value, bool), FILE_CACHE_MAXSIZE: _positive_integer, + KEEP_ATTRS: lambda choice: choice in [True, False, 'default'] } @@ -41,6 +45,17 @@ def _set_file_cache_maxsize(value): } +def _get_keep_attrs(default): + global_choice = OPTIONS['keep_attrs'] + + if global_choice is 'default': + return default + elif global_choice in [True, False]: + return global_choice + else: + raise ValueError("The global option keep_attrs must be one of True, False or 'default'.") + + class set_options(object): """Set options for xarray in a controlled context. @@ -63,8 +78,13 @@ class set_options(object): - ``cmap_divergent``: colormap to use for divergent data plots. Default: ``RdBu_r``. If string, must be matplotlib built-in colormap. Can also be a Colormap object (e.g. mpl.cm.magma) + - ``keep_attrs``: rule for whether to keep attributes on xarray + Datasets/dataarrays after operations. Either ``True`` to always keep + attrs, ``False`` to always discard them, or ``'default'`` to use original + logic that attrs should only be kept in unambiguous circumstances. + Default: ``'default'``. -f You can use ``set_options`` either as a context manager: + You can use ``set_options`` either as a context manager: >>> ds = xr.Dataset({'x': np.arange(1000)}) >>> with xr.set_options(display_width=40): diff --git a/xarray/core/resample.py b/xarray/core/resample.py index bd84e04487e..edf7dfc3d41 100644 --- a/xarray/core/resample.py +++ b/xarray/core/resample.py @@ -273,7 +273,7 @@ def apply(self, func, **kwargs): return combined.rename({self._resample_dim: self._dim}) - def reduce(self, func, dim=None, keep_attrs=False, **kwargs): + def reduce(self, func, dim=None, keep_attrs=None, **kwargs): """Reduce the items in this group by applying `func` along the pre-defined resampling dimension. diff --git a/xarray/core/variable.py b/xarray/core/variable.py index fefd48b449c..184d10b39b1 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -18,6 +18,7 @@ from .pycompat import ( OrderedDict, basestring, dask_array_type, integer_types, zip) from .utils import OrderedSet, either_dict_or_kwargs +from .options import _get_keep_attrs try: import dask.array as da @@ -1303,8 +1304,8 @@ def fillna(self, value): def where(self, cond, other=dtypes.NA): return ops.where_method(self, cond, other) - def reduce(self, func, dim=None, axis=None, keep_attrs=False, - allow_lazy=False, **kwargs): + def reduce(self, func, dim=None, axis=None, + keep_attrs=None, allow_lazy=False, **kwargs): """Reduce this array by applying `func` along some dimension(s). Parameters @@ -1351,6 +1352,8 @@ def reduce(self, func, dim=None, axis=None, keep_attrs=False, dims = [adim for n, adim in enumerate(self.dims) if n not in removed_axes] + if keep_attrs is None: + keep_attrs = _get_keep_attrs(default=False) attrs = self._attrs if keep_attrs else None return Variable(dims, data, attrs=attrs) diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index 4441375a1b1..a21ea3e6b64 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -3,8 +3,10 @@ import pytest import xarray -from xarray.core.options import OPTIONS +from xarray.core.options import OPTIONS, _get_keep_attrs from xarray.backends.file_manager import FILE_CACHE +from xarray.tests.test_dataset import create_test_data +from xarray import concat, merge def test_invalid_option_raises(): @@ -44,6 +46,18 @@ def test_file_cache_maxsize(): assert FILE_CACHE.maxsize == original_size +def test_keep_attrs(): + with pytest.raises(ValueError): + xarray.set_options(keep_attrs='invalid_str') + with xarray.set_options(keep_attrs=True): + assert OPTIONS['keep_attrs'] + with xarray.set_options(keep_attrs=False): + assert not OPTIONS['keep_attrs'] + with xarray.set_options(keep_attrs='default'): + assert _get_keep_attrs(default=True) + assert not _get_keep_attrs(default=False) + + def test_nested_options(): original = OPTIONS['display_width'] with xarray.set_options(display_width=1): @@ -52,3 +66,105 @@ def test_nested_options(): assert OPTIONS['display_width'] == 2 assert OPTIONS['display_width'] == 1 assert OPTIONS['display_width'] == original + + +def create_test_dataset_attrs(seed=0): + ds = create_test_data(seed) + ds.attrs = {'attr1': 5, 'attr2': 'history', + 'attr3': {'nested': 'more_info'}} + return ds + + +def create_test_dataarray_attrs(seed=0, var='var1'): + da = create_test_data(seed)[var] + da.attrs = {'attr1': 5, 'attr2': 'history', + 'attr3': {'nested': 'more_info'}} + return da + + +class TestAttrRetention(object): + def test_dataset_attr_retention(self): + # Use .mean() for all tests: a typical reduction operation + ds = create_test_dataset_attrs() + original_attrs = ds.attrs + + # Test default behaviour + result = ds.mean() + assert result.attrs == {} + with xarray.set_options(keep_attrs='default'): + result = ds.mean() + assert result.attrs == {} + + with xarray.set_options(keep_attrs=True): + result = ds.mean() + assert result.attrs == original_attrs + + with xarray.set_options(keep_attrs=False): + result = ds.mean() + assert result.attrs == {} + + def test_dataarray_attr_retention(self): + # Use .mean() for all tests: a typical reduction operation + da = create_test_dataarray_attrs() + original_attrs = da.attrs + + # Test default behaviour + result = da.mean() + assert result.attrs == {} + with xarray.set_options(keep_attrs='default'): + result = da.mean() + assert result.attrs == {} + + with xarray.set_options(keep_attrs=True): + result = da.mean() + assert result.attrs == original_attrs + + with xarray.set_options(keep_attrs=False): + result = da.mean() + assert result.attrs == {} + + def test_groupby_attr_retention(self): + da = xarray.DataArray([1, 2, 3], [('x', [1, 1, 2])]) + da.attrs = {'attr1': 5, 'attr2': 'history', + 'attr3': {'nested': 'more_info'}} + original_attrs = da.attrs + + # Test default behaviour + result = da.groupby('x').sum(keep_attrs=True) + assert result.attrs == original_attrs + with xarray.set_options(keep_attrs='default'): + result = da.groupby('x').sum(keep_attrs=True) + assert result.attrs == original_attrs + + with xarray.set_options(keep_attrs=True): + result1 = da.groupby('x') + result = result1.sum() + assert result.attrs == original_attrs + + with xarray.set_options(keep_attrs=False): + result = da.groupby('x').sum() + assert result.attrs == {} + + def test_concat_attr_retention(self): + ds1 = create_test_dataset_attrs() + ds2 = create_test_dataset_attrs() + ds2.attrs = {'wrong': 'attributes'} + original_attrs = ds1.attrs + + # Test default behaviour of keeping the attrs of the first + # dataset in the supplied list + # global keep_attrs option current doesn't affect concat + result = concat([ds1, ds2], dim='dim1') + assert result.attrs == original_attrs + + @pytest.mark.xfail + def test_merge_attr_retention(self): + da1 = create_test_dataarray_attrs(var='var1') + da2 = create_test_dataarray_attrs(var='var2') + da2.attrs = {'wrong': 'attributes'} + original_attrs = da1.attrs + + # merge currently discards attrs, and the global keep_attrs + # option doesn't affect this + result = merge([da1, da2]) + assert result.attrs == original_attrs From 17815b48817e69b6e5f202609d9a7568b12cdf7c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 31 Oct 2018 09:56:46 -0700 Subject: [PATCH 268/282] Raise more informative error when converting tuples to Variable. (#2523) Fixes #1016 --- xarray/core/variable.py | 8 ++++---- xarray/tests/test_dataset.py | 2 +- xarray/tests/test_variable.py | 4 +++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 184d10b39b1..0bff06e7546 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -77,11 +77,11 @@ def as_variable(obj, name=None): elif isinstance(obj, tuple): try: obj = Variable(*obj) - except TypeError: + except (TypeError, ValueError) as error: # use .format() instead of % because it handles tuples consistently - raise TypeError('tuples to convert into variables must be of the ' - 'form (dims, data[, attrs, encoding]): ' - '{}'.format(obj)) + raise error.__class__('Could not convert tuple of form ' + '(dims, data[, attrs, encoding]): ' + '{} to Variable.'.format(obj)) elif utils.is_scalar(obj): obj = Variable([], obj) elif isinstance(obj, (pd.Index, IndexVariable)) and obj.name is not None: diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index aa226ff1ce8..a32253c19e5 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -242,7 +242,7 @@ def test_constructor(self): Dataset({'a': x1, 'b': x2}) with raises_regex(ValueError, "disallows such variables"): Dataset({'a': x1, 'x': z}) - with raises_regex(TypeError, 'tuples to convert'): + with raises_regex(TypeError, 'tuple of form'): Dataset({'x': (1, 2, 3, 4, 5, 6, 7)}) with raises_regex(ValueError, 'already exists as a scalar'): Dataset({'x': 0, 'y': ('x', [1, 2, 3])}) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 3753221f352..0bd440781ac 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -970,8 +970,10 @@ def test_as_variable(self): expected_extra.attrs, expected_extra.encoding) assert_identical(expected_extra, as_variable(xarray_tuple)) - with raises_regex(TypeError, 'tuples to convert'): + with raises_regex(TypeError, 'tuple of form'): as_variable(tuple(data)) + with raises_regex(ValueError, 'tuple of form'): # GH1016 + as_variable(('five', 'six', 'seven')) with raises_regex( TypeError, 'without an explicit list of dimensions'): as_variable(data) From 656f8bd05e44880c21c1ad56a03cfd1b4d0f38ee Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Thu, 1 Nov 2018 01:04:25 -0400 Subject: [PATCH 269/282] Switch enable_cftimeindex to True by default (#2516) * Switch enable_cftimeindex to True by default * Add a friendlier error message when plotting cftime objects * Mention that the non-standard calendars are used in climate science * Add GH issue references to docs * Deprecate enable_cftimeindex option * Add CFTimeIndex.to_datetimeindex method * Add friendlier error message for resample * lint * Address review comments * Take into account microsecond attribute in cftime_to_nptime * Add test for decoding dates with microsecond-resolution units This would have failed before including the microsecond attribute of each date in cftime_to_nptime in eaa4a44. * Fix typo in time-series.rst * Formatting * Fix test_decode_cf_datetime_non_iso_strings * Prevent warning emitted from set_options.__exit__ --- doc/api-hidden.rst | 1 + doc/time-series.rst | 102 ++++---- doc/whats-new.rst | 16 ++ xarray/coding/cftimeindex.py | 53 +++++ xarray/coding/times.py | 60 +++-- xarray/core/common.py | 31 +++ xarray/core/options.py | 19 +- xarray/core/utils.py | 16 +- xarray/plot/plot.py | 9 +- xarray/tests/test_backends.py | 64 ++--- xarray/tests/test_cftimeindex.py | 61 +++-- xarray/tests/test_coding_times.py | 376 ++++++++++++------------------ xarray/tests/test_dataarray.py | 7 +- xarray/tests/test_options.py | 5 +- xarray/tests/test_utils.py | 14 +- 15 files changed, 447 insertions(+), 387 deletions(-) diff --git a/doc/api-hidden.rst b/doc/api-hidden.rst index 0d8189d21cf..4b2fed8be37 100644 --- a/doc/api-hidden.rst +++ b/doc/api-hidden.rst @@ -152,3 +152,4 @@ plot.FacetGrid.map CFTimeIndex.shift + CFTimeIndex.to_datetimeindex diff --git a/doc/time-series.rst b/doc/time-series.rst index c1a686b409f..7befd954f35 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -71,10 +71,11 @@ One unfortunate limitation of using ``datetime64[ns]`` is that it limits the native representation of dates to those that fall between the years 1678 and 2262. When a netCDF file contains dates outside of these bounds, dates will be returned as arrays of :py:class:`cftime.datetime` objects and a :py:class:`~xarray.CFTimeIndex` -can be used for indexing. The :py:class:`~xarray.CFTimeIndex` enables only a subset of -the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only enabled -when using the standalone version of ``cftime`` (not the version packaged with -earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more information. +will be used for indexing. :py:class:`~xarray.CFTimeIndex` enables a subset of +the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only +fully compatible with the standalone version of ``cftime`` (not the version +packaged with earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more +information. Datetime indexing ----------------- @@ -221,18 +222,28 @@ Non-standard calendars and dates outside the Timestamp-valid range Through the standalone ``cftime`` library and a custom subclass of :py:class:`pandas.Index`, xarray supports a subset of the indexing functionality enabled through the standard :py:class:`pandas.DatetimeIndex` for -dates from non-standard calendars or dates using a standard calendar, but -outside the `Timestamp-valid range`_ (approximately between years 1678 and -2262). This behavior has not yet been turned on by default; to take advantage -of this functionality, you must have the ``enable_cftimeindex`` option set to -``True`` within your context (see :py:func:`~xarray.set_options` for more -information). It is expected that this will become the default behavior in -xarray version 0.11. - -For instance, you can create a DataArray indexed by a time -coordinate with a no-leap calendar within a context manager setting the -``enable_cftimeindex`` option, and the time index will be cast to a -:py:class:`~xarray.CFTimeIndex`: +dates from non-standard calendars commonly used in climate science or dates +using a standard calendar, but outside the `Timestamp-valid range`_ +(approximately between years 1678 and 2262). + +.. note:: + + As of xarray version 0.11, by default, :py:class:`cftime.datetime` objects + will be used to represent times (either in indexes, as a + :py:class:`~xarray.CFTimeIndex`, or in data arrays with dtype object) if + any of the following are true: + + - The dates are from a non-standard calendar + - Any dates are outside the Timestamp-valid range. + + Otherwise pandas-compatible dates from a standard calendar will be + represented with the ``np.datetime64[ns]`` data type, enabling the use of a + :py:class:`pandas.DatetimeIndex` or arrays with dtype ``np.datetime64[ns]`` + and their full set of associated features. + +For example, you can create a DataArray indexed by a time +coordinate with dates from a no-leap calendar and a +:py:class:`~xarray.CFTimeIndex` will automatically be used: .. ipython:: python @@ -241,27 +252,11 @@ coordinate with a no-leap calendar within a context manager setting the dates = [DatetimeNoLeap(year, month, 1) for year, month in product(range(1, 3), range(1, 13))] - with xr.set_options(enable_cftimeindex=True): - da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], - name='foo') + da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], name='foo') -.. note:: - - With the ``enable_cftimeindex`` option activated, a :py:class:`~xarray.CFTimeIndex` - will be used for time indexing if any of the following are true: - - - The dates are from a non-standard calendar - - Any dates are outside the Timestamp-valid range - - Otherwise a :py:class:`pandas.DatetimeIndex` will be used. In addition, if any - variable (not just an index variable) is encoded using a non-standard - calendar, its times will be decoded into :py:class:`cftime.datetime` objects, - regardless of whether or not they can be represented using - ``np.datetime64[ns]`` objects. - xarray also includes a :py:func:`~xarray.cftime_range` function, which enables -creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates. For instance, we can -create the same dates and DataArray we created above using: +creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates. For +instance, we can create the same dates and DataArray we created above using: .. ipython:: python @@ -317,13 +312,42 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: .. ipython:: python - da.to_netcdf('example.nc') - xr.open_dataset('example.nc') + da.to_netcdf('example-no-leap.nc') + xr.open_dataset('example-no-leap.nc') .. note:: - Currently resampling along the time dimension for data indexed by a - :py:class:`~xarray.CFTimeIndex` is not supported. + While much of the time series functionality that is possible for standard + dates has been implemented for dates from non-standard calendars, there are + still some remaining important features that have yet to be implemented, + for example: + + - Resampling along the time dimension for data indexed by a + :py:class:`~xarray.CFTimeIndex` (:issue:`2191`, :issue:`2458`) + - Built-in plotting of data with :py:class:`cftime.datetime` coordinate axes + (:issue:`2164`). + + For some use-cases it may still be useful to convert from + a :py:class:`~xarray.CFTimeIndex` to a :py:class:`pandas.DatetimeIndex`, + despite the difference in calendar types (e.g. to allow the use of some + forms of resample with non-standard calendars). The recommended way of + doing this is to use the built-in + :py:meth:`~xarray.CFTimeIndex.to_datetimeindex` method: + + .. ipython:: python + + modern_times = xr.cftime_range('2000', periods=24, freq='MS', calendar='noleap') + da = xr.DataArray(range(24), [('time', modern_times)]) + da + datetimeindex = da.indexes['time'].to_datetimeindex() + da['time'] = datetimeindex + da.resample(time='Y').mean('time') + + However in this case one should use caution to only perform operations which + do not depend on differences between dates (e.g. differentiation, + interpolation, or upsampling with resample), as these could introduce subtle + and silent errors due to the difference in calendar types between the dates + encoded in your data and the dates stored in memory. .. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timestamp-limitations .. _ISO 8601-format: https://en.wikipedia.org/wiki/ISO_8601 diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 2ffbc60622d..9db3d35af84 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,6 +33,22 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ +- For non-standard calendars commonly used in climate science, xarray will now + always use :py:class:`cftime.datetime` objects, rather than by default try to + coerce them to ``np.datetime64[ns]`` objects. A + :py:class:`~xarray.CFTimeIndex` will be used for indexing along time + coordinates in these cases. A new method, + :py:meth:`~xarray.CFTimeIndex.to_datetimeindex`, has been added + to aid in converting from a :py:class:`~xarray.CFTimeIndex` to a + :py:class:`pandas.DatetimeIndex` for the remaining use-cases where + using a :py:class:`~xarray.CFTimeIndex` is still a limitation (e.g. for + resample or plotting). Setting the ``enable_cftimeindex`` option is now a + no-op and emits a ``FutureWarning``. +- ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. + Call :py:meth:`Dataset.transpose` directly instead. +- Iterating over a ``Dataset`` now includes only data variables, not coordinates. + Similarily, calling ``len`` and ``bool`` on a ``Dataset`` now + includes only data variables - Finished deprecation cycles: - ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. Call :py:meth:`Dataset.transpose` directly instead. diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 5de055c1b9a..2ce996b2bd2 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -42,6 +42,7 @@ from __future__ import absolute_import import re +import warnings from datetime import timedelta import numpy as np @@ -50,6 +51,8 @@ from xarray.core import pycompat from xarray.core.utils import is_scalar +from .times import cftime_to_nptime, infer_calendar_name, _STANDARD_CALENDARS + def named(name, pattern): return '(?P<' + name + '>' + pattern + ')' @@ -381,6 +384,56 @@ def _add_delta(self, deltas): # pandas. No longer used as of pandas 0.23. return self + deltas + def to_datetimeindex(self, unsafe=False): + """If possible, convert this index to a pandas.DatetimeIndex. + + Parameters + ---------- + unsafe : bool + Flag to turn off warning when converting from a CFTimeIndex with + a non-standard calendar to a DatetimeIndex (default ``False``). + + Returns + ------- + pandas.DatetimeIndex + + Raises + ------ + ValueError + If the CFTimeIndex contains dates that are not possible in the + standard calendar or outside the pandas.Timestamp-valid range. + + Warns + ----- + RuntimeWarning + If converting from a non-standard calendar to a DatetimeIndex. + + Warnings + -------- + Note that for non-standard calendars, this will change the calendar + type of the index. In that case the result of this method should be + used with caution. + + Examples + -------- + >>> import xarray as xr + >>> times = xr.cftime_range('2000', periods=2, calendar='gregorian') + >>> times + CFTimeIndex([2000-01-01 00:00:00, 2000-01-02 00:00:00], dtype='object') + >>> times.to_datetimeindex() + DatetimeIndex(['2000-01-01', '2000-01-02'], dtype='datetime64[ns]', freq=None) + """ # noqa: E501 + nptimes = cftime_to_nptime(self) + calendar = infer_calendar_name(self) + if calendar not in _STANDARD_CALENDARS and not unsafe: + warnings.warn( + 'Converting a CFTimeIndex with dates from a non-standard ' + 'calendar, {!r}, to a pandas.DatetimeIndex, which uses dates ' + 'from the standard calendar. This may lead to subtle errors ' + 'in operations that depend on the length of time between ' + 'dates.'.format(calendar), RuntimeWarning) + return pd.DatetimeIndex(nptimes) + def _parse_iso8601_without_reso(date_type, datetime_str): date, _ = _parse_iso8601_with_reso(date_type, datetime_str) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 16380976def..dfc4b2fb023 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -12,7 +12,6 @@ from ..core import indexing from ..core.common import contains_cftime_datetimes from ..core.formatting import first_n_items, format_timestamp, last_item -from ..core.options import OPTIONS from ..core.pycompat import PY3 from ..core.variable import Variable from .variables import ( @@ -61,8 +60,9 @@ def _require_standalone_cftime(): try: import cftime # noqa: F401 except ImportError: - raise ImportError('Using a CFTimeIndex requires the standalone ' - 'version of the cftime library.') + raise ImportError('Decoding times with non-standard calendars ' + 'or outside the pandas.Timestamp-valid range ' + 'requires the standalone cftime package.') def _netcdf_to_numpy_timeunit(units): @@ -84,41 +84,32 @@ def _unpack_netcdf_time_units(units): return delta_units, ref_date -def _decode_datetime_with_cftime(num_dates, units, calendar, - enable_cftimeindex): +def _decode_datetime_with_cftime(num_dates, units, calendar): cftime = _import_cftime() - if enable_cftimeindex: - _require_standalone_cftime() + + if cftime.__name__ == 'cftime': dates = np.asarray(cftime.num2date(num_dates, units, calendar, only_use_cftime_datetimes=True)) else: + # Must be using num2date from an old version of netCDF4 which + # does not have the only_use_cftime_datetimes option. dates = np.asarray(cftime.num2date(num_dates, units, calendar)) if (dates[np.nanargmin(num_dates)].year < 1678 or dates[np.nanargmax(num_dates)].year >= 2262): - if not enable_cftimeindex or calendar in _STANDARD_CALENDARS: + if calendar in _STANDARD_CALENDARS: warnings.warn( 'Unable to decode time axis into full ' 'numpy.datetime64 objects, continuing using dummy ' 'cftime.datetime objects instead, reason: dates out ' 'of range', SerializationWarning, stacklevel=3) else: - if enable_cftimeindex: - if calendar in _STANDARD_CALENDARS: - dates = cftime_to_nptime(dates) - else: - try: - dates = cftime_to_nptime(dates) - except ValueError as e: - warnings.warn( - 'Unable to decode time axis into full ' - 'numpy.datetime64 objects, continuing using ' - 'dummy cftime.datetime objects instead, reason:' - '{0}'.format(e), SerializationWarning, stacklevel=3) + if calendar in _STANDARD_CALENDARS: + dates = cftime_to_nptime(dates) return dates -def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): +def _decode_cf_datetime_dtype(data, units, calendar): # Verify that at least the first and last date can be decoded # successfully. Otherwise, tracebacks end up swallowed by # Dataset.__repr__ when users try to view their lazily decoded array. @@ -128,8 +119,7 @@ def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): last_item(values) or [0]]) try: - result = decode_cf_datetime(example_value, units, calendar, - enable_cftimeindex) + result = decode_cf_datetime(example_value, units, calendar) except Exception: calendar_msg = ('the default calendar' if calendar is None else 'calendar %r' % calendar) @@ -145,8 +135,7 @@ def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): return dtype -def decode_cf_datetime(num_dates, units, calendar=None, - enable_cftimeindex=False): +def decode_cf_datetime(num_dates, units, calendar=None): """Given an array of numeric dates in netCDF format, convert it into a numpy array of date time objects. @@ -200,8 +189,7 @@ def decode_cf_datetime(num_dates, units, calendar=None, except (OutOfBoundsDatetime, OverflowError): dates = _decode_datetime_with_cftime( - flat_num_dates.astype(np.float), units, calendar, - enable_cftimeindex) + flat_num_dates.astype(np.float), units, calendar) return dates.reshape(num_dates.shape) @@ -291,7 +279,16 @@ def cftime_to_nptime(times): times = np.asarray(times) new = np.empty(times.shape, dtype='M8[ns]') for i, t in np.ndenumerate(times): - dt = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second) + try: + # Use pandas.Timestamp in place of datetime.datetime, because + # NumPy casts it safely it np.datetime64[ns] for dates outside + # 1678 to 2262 (this is not currently the case for + # datetime.datetime). + dt = pd.Timestamp(t.year, t.month, t.day, t.hour, t.minute, + t.second, t.microsecond) + except ValueError as e: + raise ValueError('Cannot convert date {} to a date in the ' + 'standard calendar. Reason: {}.'.format(t, e)) new[i] = np.datetime64(dt) return new @@ -404,15 +401,12 @@ def encode(self, variable, name=None): def decode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_decoding(variable) - enable_cftimeindex = OPTIONS['enable_cftimeindex'] if 'units' in attrs and 'since' in attrs['units']: units = pop_to(attrs, encoding, 'units') calendar = pop_to(attrs, encoding, 'calendar') - dtype = _decode_cf_datetime_dtype( - data, units, calendar, enable_cftimeindex) + dtype = _decode_cf_datetime_dtype(data, units, calendar) transform = partial( - decode_cf_datetime, units=units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + decode_cf_datetime, units=units, calendar=calendar) data = lazy_elemwise_func(data, transform, dtype) return Variable(dims, data, attrs, encoding) diff --git a/xarray/core/common.py b/xarray/core/common.py index e303c485523..508a19b7115 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -659,6 +659,7 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, from .dataarray import DataArray from .resample import RESAMPLE_DIM + from ..coding.cftimeindex import CFTimeIndex if keep_attrs is None: keep_attrs = _get_keep_attrs(default=False) @@ -687,6 +688,20 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, else: raise TypeError("Dimension name should be a string; " "was passed %r" % dim) + + if isinstance(self.indexes[dim_name], CFTimeIndex): + raise NotImplementedError( + 'Resample is currently not supported along a dimension ' + 'indexed by a CFTimeIndex. For certain kinds of downsampling ' + 'it may be possible to work around this by converting your ' + 'time index to a DatetimeIndex using ' + 'CFTimeIndex.to_datetimeindex. Use caution when doing this ' + 'however, because switching to a DatetimeIndex from a ' + 'CFTimeIndex with a non-standard calendar entails a change ' + 'in the calendar type, which could lead to subtle and silent ' + 'errors.' + ) + group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base) resampler = self._resample_cls(self, group=group, dim=dim_name, @@ -700,6 +715,8 @@ def _resample_immediately(self, freq, dim, how, skipna, """Implement the original version of .resample() which immediately executes the desired resampling operation. """ from .dataarray import DataArray + from ..coding.cftimeindex import CFTimeIndex + RESAMPLE_DIM = '__resample_dim__' warnings.warn("\n.resample() has been modified to defer " @@ -709,8 +726,22 @@ def _resample_immediately(self, freq, dim, how, skipna, dim=dim, freq=freq, how=how), FutureWarning, stacklevel=3) + if isinstance(self.indexes[dim], CFTimeIndex): + raise NotImplementedError( + 'Resample is currently not supported along a dimension ' + 'indexed by a CFTimeIndex. For certain kinds of downsampling ' + 'it may be possible to work around this by converting your ' + 'time index to a DatetimeIndex using ' + 'CFTimeIndex.to_datetimeindex. Use caution when doing this ' + 'however, because switching to a DatetimeIndex from a ' + 'CFTimeIndex with a non-standard calendar entails a change ' + 'in the calendar type, which could lead to subtle and silent ' + 'errors.' + ) + if isinstance(dim, basestring): dim = self[dim] + group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) grouper = pd.Grouper(freq=freq, how=how, closed=closed, label=label, base=base) diff --git a/xarray/core/options.py b/xarray/core/options.py index eb3013d5233..ab461ca86bc 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, division, print_function +import warnings + DISPLAY_WIDTH = 'display_width' ARITHMETIC_JOIN = 'arithmetic_join' ENABLE_CFTIMEINDEX = 'enable_cftimeindex' @@ -12,7 +14,7 @@ OPTIONS = { DISPLAY_WIDTH: 80, ARITHMETIC_JOIN: 'inner', - ENABLE_CFTIMEINDEX: False, + ENABLE_CFTIMEINDEX: True, FILE_CACHE_MAXSIZE: 128, CMAP_SEQUENTIAL: 'viridis', CMAP_DIVERGENT: 'RdBu_r', @@ -40,8 +42,16 @@ def _set_file_cache_maxsize(value): FILE_CACHE.maxsize = value +def _warn_on_setting_enable_cftimeindex(enable_cftimeindex): + warnings.warn( + 'The enable_cftimeindex option is now a no-op ' + 'and will be removed in a future version of xarray.', + FutureWarning) + + _SETTERS = { FILE_CACHE_MAXSIZE: _set_file_cache_maxsize, + ENABLE_CFTIMEINDEX: _warn_on_setting_enable_cftimeindex } @@ -65,9 +75,6 @@ class set_options(object): Default: ``80``. - ``arithmetic_join``: DataArray/Dataset alignment in binary operations. Default: ``'inner'``. - - ``enable_cftimeindex``: flag to enable using a ``CFTimeIndex`` - for time indexes with non-standard calendars or dates outside the - Timestamp-valid range. Default: ``False``. - ``file_cache_maxsize``: maximum number of open files to hold in xarray's global least-recently-usage cached. This should be smaller than your system's per-process file descriptor limit, e.g., ``ulimit -n`` on Linux. @@ -102,7 +109,7 @@ class set_options(object): """ def __init__(self, **kwargs): - self.old = OPTIONS.copy() + self.old = {} for k, v in kwargs.items(): if k not in OPTIONS: raise ValueError( @@ -111,6 +118,7 @@ def __init__(self, **kwargs): if k in _VALIDATORS and not _VALIDATORS[k](v): raise ValueError( 'option %r given an invalid value: %r' % (k, v)) + self.old[k] = OPTIONS[k] self._apply_update(kwargs) def _apply_update(self, options_dict): @@ -123,5 +131,4 @@ def __enter__(self): return def __exit__(self, type, value, traceback): - OPTIONS.clear() self._apply_update(self.old) diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 5c9d8bfbf77..015916d668e 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -13,7 +13,6 @@ import numpy as np import pandas as pd -from .options import OPTIONS from .pycompat import ( OrderedDict, basestring, bytes_type, dask_array_type, iteritems) @@ -41,16 +40,13 @@ def wrapper(*args, **kwargs): def _maybe_cast_to_cftimeindex(index): from ..coding.cftimeindex import CFTimeIndex - if not OPTIONS['enable_cftimeindex']: - return index - else: - if index.dtype == 'O': - try: - return CFTimeIndex(index) - except (ImportError, TypeError): - return index - else: + if index.dtype == 'O': + try: + return CFTimeIndex(index) + except (ImportError, TypeError): return index + else: + return index def safe_cast_to_index(array): diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 7129157ec7f..8d21e084946 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -145,8 +145,13 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, hue=None, darray = darray.squeeze() if contains_cftime_datetimes(darray): - raise NotImplementedError('Plotting arrays of cftime.datetime objects ' - 'is currently not possible.') + raise NotImplementedError( + 'Built-in plotting of arrays of cftime.datetime objects or arrays ' + 'indexed by cftime.datetime objects is currently not implemented ' + 'within xarray. A possible workaround is to use the ' + 'nc-time-axis package ' + '(https://github.com/SciTools/nc-time-axis) to convert the dates ' + 'to a plottable type and plot your data directly with matplotlib.') plot_dims = set(darray.dims) plot_dims.discard(row) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index c6a2df733fa..80d3a9d526e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -356,7 +356,7 @@ def test_roundtrip_numpy_datetime_data(self): assert actual.t0.encoding['units'] == 'days since 1950-01-01' @requires_cftime - def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + def test_roundtrip_cftime_datetime_data(self): from .test_coding_times import _all_cftime_date_types date_types = _all_cftime_date_types() @@ -373,21 +373,20 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): warnings.filterwarnings( 'ignore', 'Unable to decode time axis') - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected, save_kwargs=kwds) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t.encoding['units'] == - 'days since 0001-01-01 00:00:00.000000') - assert (actual.t.encoding['calendar'] == - expected_calendar) - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t0.encoding['units'] == - 'days since 0001-01-01') - assert (actual.t.encoding['calendar'] == - expected_calendar) + with self.roundtrip(expected, save_kwargs=kwds) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t.encoding['units'] == + 'days since 0001-01-01 00:00:00.000000') + assert (actual.t.encoding['calendar'] == + expected_calendar) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t0.encoding['units'] == + 'days since 0001-01-01') + assert (actual.t.encoding['calendar'] == + expected_calendar) def test_roundtrip_timedelta_data(self): time_deltas = pd.to_timedelta(['1h', '2h', 'NaT']) @@ -2087,7 +2086,7 @@ def test_roundtrip_numpy_datetime_data(self): with self.roundtrip(expected) as actual: assert_identical(expected, actual) - def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + def test_roundtrip_cftime_datetime_data(self): # Override method in DatasetIOBase - remove not applicable # save_kwds from .test_coding_times import _all_cftime_date_types @@ -2099,33 +2098,12 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): expected_decoded_t = np.array(times) expected_decoded_t0 = np.array([date_type(1, 1, 1)]) - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - - def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): - # Override method in DatasetIOBase - remove not applicable - # save_kwds - from .test_coding_times import _all_cftime_date_types - - date_types = _all_cftime_date_types() - for date_type in date_types.values(): - times = [date_type(1, 1, 1), date_type(1, 1, 2)] - expected = Dataset({'t': ('t', times), 't0': times[0]}) - expected_decoded_t = np.array(times) - expected_decoded_t0 = np.array([date_type(1, 1, 1)]) - - with xr.set_options(enable_cftimeindex=False): - with self.roundtrip(expected) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() + with self.roundtrip(expected) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() def test_write_store(self): # Override method in DatasetIOBase - not applicable to dask diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index e18c55d2fae..5e710827ff8 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -13,7 +13,8 @@ from xarray.tests import assert_array_equal, assert_identical from . import has_cftime, has_cftime_or_netCDF4, requires_cftime -from .test_coding_times import _all_cftime_date_types +from .test_coding_times import (_all_cftime_date_types, _ALL_CALENDARS, + _NON_STANDARD_CALENDARS) def date_dict(year=None, month=None, day=None, @@ -360,7 +361,7 @@ def test_groupby(da): @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_resample_error(da): - with pytest.raises(TypeError): + with pytest.raises(NotImplementedError, match='to_datetimeindex'): da.resample(time='Y') @@ -594,18 +595,16 @@ def test_indexing_in_dataframe_iloc(df, index): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_concat_cftimeindex(date_type, enable_cftimeindex): - with xr.set_options(enable_cftimeindex=enable_cftimeindex): - da1 = xr.DataArray( - [1., 2.], coords=[[date_type(1, 1, 1), date_type(1, 2, 1)]], - dims=['time']) - da2 = xr.DataArray( - [3., 4.], coords=[[date_type(1, 3, 1), date_type(1, 4, 1)]], - dims=['time']) - da = xr.concat([da1, da2], dim='time') - - if enable_cftimeindex and has_cftime: +def test_concat_cftimeindex(date_type): + da1 = xr.DataArray( + [1., 2.], coords=[[date_type(1, 1, 1), date_type(1, 2, 1)]], + dims=['time']) + da2 = xr.DataArray( + [3., 4.], coords=[[date_type(1, 3, 1), date_type(1, 4, 1)]], + dims=['time']) + da = xr.concat([da1, da2], dim='time') + + if has_cftime: assert isinstance(da.indexes['time'], CFTimeIndex) else: assert isinstance(da.indexes['time'], pd.Index) @@ -746,3 +745,37 @@ def test_parse_array_of_cftime_strings(): expected = np.array(DatetimeNoLeap(2000, 1, 1)) result = _parse_array_of_cftime_strings(strings, DatetimeNoLeap) np.testing.assert_array_equal(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +@pytest.mark.parametrize('unsafe', [False, True]) +def test_to_datetimeindex(calendar, unsafe): + index = xr.cftime_range('2000', periods=5, calendar=calendar) + expected = pd.date_range('2000', periods=5) + + if calendar in _NON_STANDARD_CALENDARS and not unsafe: + with pytest.warns(RuntimeWarning, match='non-standard'): + result = index.to_datetimeindex() + else: + result = index.to_datetimeindex() + + assert result.equals(expected) + np.testing.assert_array_equal(result, expected) + assert isinstance(result, pd.DatetimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_to_datetimeindex_out_of_range(calendar): + index = xr.cftime_range('0001', periods=5, calendar=calendar) + with pytest.raises(ValueError, match='0001'): + index.to_datetimeindex() + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', ['all_leap', '360_day']) +def test_to_datetimeindex_feb_29(calendar): + index = xr.cftime_range('2001-02-28', periods=2, calendar=calendar) + with pytest.raises(ValueError, match='29'): + index.to_datetimeindex() diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index f76b8c3ceab..0ca57f98a6d 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -7,10 +7,9 @@ import pandas as pd import pytest -from xarray import DataArray, Variable, coding, decode_cf, set_options -from xarray.coding.times import (_import_cftime, decode_cf_datetime, - encode_cf_datetime) -from xarray.coding.variables import SerializationWarning +from xarray import DataArray, Variable, coding, decode_cf +from xarray.coding.times import (_import_cftime, cftime_to_nptime, + decode_cf_datetime, encode_cf_datetime) from xarray.core.common import contains_cftime_datetimes from . import ( @@ -48,21 +47,14 @@ ([0.5, 1.5], 'hours since 1900-01-01T00:00:00'), (0, 'milliseconds since 2000-01-01T00:00:00'), (0, 'microseconds since 2000-01-01T00:00:00'), - (np.int32(788961600), 'seconds since 1981-01-01') # GH2002 + (np.int32(788961600), 'seconds since 1981-01-01'), # GH2002 + (12300 + np.arange(5), 'hour since 1680-01-01 00:00:00.500000') ] _CF_DATETIME_TESTS = [num_dates_units + (calendar,) for num_dates_units, calendar in product(_CF_DATETIME_NUM_DATES_UNITS, _STANDARD_CALENDARS)] -@np.vectorize -def _ensure_naive_tz(dt): - if hasattr(dt, 'tzinfo'): - return dt.replace(tzinfo=None) - else: - return dt - - def _all_cftime_date_types(): try: import cftime @@ -83,24 +75,27 @@ def _all_cftime_date_types(): _CF_DATETIME_TESTS) def test_cf_datetime(num_dates, units, calendar): cftime = _import_cftime() - expected = _ensure_naive_tz( - cftime.num2date(num_dates, units, calendar)) + if cftime.__name__ == 'cftime': + expected = cftime.num2date(num_dates, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(num_dates, units, calendar) + min_y = np.ravel(np.atleast_1d(expected))[np.nanargmin(num_dates)].year + max_y = np.ravel(np.atleast_1d(expected))[np.nanargmax(num_dates)].year + if min_y >= 1678 and max_y < 2262: + expected = cftime_to_nptime(expected) + with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime(num_dates, units, calendar) - if (isinstance(actual, np.ndarray) and - np.issubdtype(actual.dtype, np.datetime64)): - # self.assertEqual(actual.dtype.kind, 'M') - # For some reason, numpy 1.8 does not compare ns precision - # datetime64 arrays as equal to arrays of datetime objects, - # but it works for us precision. Thus, convert to us - # precision for the actual array equal comparison... - actual_cmp = actual.astype('M8[us]') - else: - actual_cmp = actual - assert_array_equal(expected, actual_cmp) + + abs_diff = np.atleast_1d(abs(actual - expected)).astype(np.timedelta64) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() encoded, _, _ = coding.times.encode_cf_datetime(actual, units, calendar) if '1-1-1' not in units: @@ -124,8 +119,12 @@ def test_decode_cf_datetime_overflow(): # checks for # https://github.com/pydata/pandas/issues/14068 # https://github.com/pydata/xarray/issues/975 + try: + from cftime import DatetimeGregorian + except ImportError: + from netcdftime import DatetimeGregorian - from datetime import datetime + datetime = DatetimeGregorian units = 'days since 2000-01-01 00:00:00' # date after 2262 and before 1678 @@ -151,39 +150,32 @@ def test_decode_cf_datetime_non_standard_units(): @requires_cftime_or_netCDF4 def test_decode_cf_datetime_non_iso_strings(): # datetime strings that are _almost_ ISO compliant but not quite, - # but which netCDF4.num2date can still parse correctly + # but which cftime.num2date can still parse correctly expected = pd.date_range(periods=100, start='2000-01-01', freq='h') cases = [(np.arange(100), 'hours since 2000-01-01 0'), (np.arange(100), 'hours since 2000-1-1 0'), (np.arange(100), 'hours since 2000-01-01 0:00')] for num_dates, units in cases: actual = coding.times.decode_cf_datetime(num_dates, units) - assert_array_equal(actual, expected) + abs_diff = abs(actual - expected.values) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) -def test_decode_standard_calendar_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) +def test_decode_standard_calendar_inside_timestamp_range(calendar): cftime = _import_cftime() + units = 'days since 0001-01-01' - times = pd.date_range('2001-04-01-00', end='2001-04-30-23', - freq='H') - noleap_time = cftime.date2num(times.to_pydatetime(), units, - calendar=calendar) + times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') + time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar) expected = times.values expected_dtype = np.dtype('M8[ns]') - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + actual = coding.times.decode_cf_datetime(time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -193,32 +185,28 @@ def test_decode_standard_calendar_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_non_standard_calendar_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') - noleap_time = cftime.date2num(times.to_pydatetime(), units, - calendar=calendar) - if enable_cftimeindex: - expected = cftime.num2date(noleap_time, units, calendar=calendar) - expected_dtype = np.dtype('O') + non_standard_time = cftime.date2num( + times.to_pydatetime(), units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date( + non_standard_time, units, calendar=calendar, + only_use_cftime_datetimes=True) else: - expected = times.values - expected_dtype = np.dtype('M8[ns]') + expected = cftime.num2date(non_standard_time, units, + calendar=calendar) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + expected_dtype = np.dtype('O') + + actual = coding.times.decode_cf_datetime( + non_standard_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -228,33 +216,27 @@ def test_decode_non_standard_calendar_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) -def test_decode_dates_outside_timestamp_range( - calendar, enable_cftimeindex): +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_decode_dates_outside_timestamp_range(calendar): from datetime import datetime - - if enable_cftimeindex: - pytest.importorskip('cftime') - cftime = _import_cftime() units = 'days since 0001-01-01' times = [datetime(1, 4, 1, h) for h in range(1, 5)] - noleap_time = cftime.date2num(times, units, calendar=calendar) - if enable_cftimeindex: - expected = cftime.num2date(noleap_time, units, calendar=calendar, + time = cftime.date2num(times, units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(time, units, calendar=calendar, only_use_cftime_datetimes=True) else: - expected = cftime.num2date(noleap_time, units, calendar=calendar) + expected = cftime.num2date(time, units, calendar=calendar) + expected_date_type = type(expected[0]) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + time, units, calendar=calendar) assert all(isinstance(value, expected_date_type) for value in actual) abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -264,57 +246,37 @@ def test_decode_dates_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) def test_decode_standard_calendar_single_element_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): units = 'days since 0001-01-01' for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + num_time, units, calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_non_standard_calendar_single_element_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): units = 'days since 0001-01-01' for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - if enable_cftimeindex: - assert actual.dtype == np.dtype('O') - else: - assert actual.dtype == np.dtype('M8[ns]') + num_time, units, calendar=calendar) + assert actual.dtype == np.dtype('O') @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_single_element_outside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' for days in [1, 1470376]: @@ -323,40 +285,39 @@ def test_decode_single_element_outside_timestamp_range( warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - expected = cftime.num2date(days, units, calendar) + num_time, units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(days, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(days, units, calendar) + assert isinstance(actual.item(), type(expected)) @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) def test_decode_standard_calendar_multidim_time_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = cftime.date2num(times1.to_pydatetime(), - units, calendar=calendar) - noleap_time2 = cftime.date2num(times2.to_pydatetime(), - units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 + time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 expected1 = times1.values expected2 = times2.values actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') abs_diff1 = abs(actual[:, 0] - expected1) @@ -369,39 +330,35 @@ def test_decode_standard_calendar_multidim_time_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = cftime.date2num(times1.to_pydatetime(), - units, calendar=calendar) - noleap_time2 = cftime.date2num(times2.to_pydatetime(), - units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 - - if enable_cftimeindex: - expected1 = cftime.num2date(noleap_time1, units, calendar) - expected2 = cftime.num2date(noleap_time2, units, calendar) - expected_dtype = np.dtype('O') + time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 + + if cftime.__name__ == 'cftime': + expected1 = cftime.num2date(time1, units, calendar, + only_use_cftime_datetimes=True) + expected2 = cftime.num2date(time2, units, calendar, + only_use_cftime_datetimes=True) else: - expected1 = times1.values - expected2 = times2.values - expected_dtype = np.dtype('M8[ns]') + expected1 = cftime.num2date(time1, units, calendar) + expected2 = cftime.num2date(time2, units, calendar) + + expected_dtype = np.dtype('O') actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff1 = abs(actual[:, 0] - expected1) @@ -414,41 +371,34 @@ def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) def test_decode_multidim_time_outside_timestamp_range( - calendar, enable_cftimeindex): + calendar): from datetime import datetime - - if enable_cftimeindex: - pytest.importorskip('cftime') - cftime = _import_cftime() units = 'days since 0001-01-01' times1 = [datetime(1, 4, day) for day in range(1, 6)] times2 = [datetime(1, 5, day) for day in range(1, 6)] - noleap_time1 = cftime.date2num(times1, units, calendar=calendar) - noleap_time2 = cftime.date2num(times2, units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 - - if enable_cftimeindex: - expected1 = cftime.num2date(noleap_time1, units, calendar, + time1 = cftime.date2num(times1, units, calendar=calendar) + time2 = cftime.date2num(times2, units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 + + if cftime.__name__ == 'cftime': + expected1 = cftime.num2date(time1, units, calendar, only_use_cftime_datetimes=True) - expected2 = cftime.num2date(noleap_time2, units, calendar, + expected2 = cftime.num2date(time2, units, calendar, only_use_cftime_datetimes=True) else: - expected1 = cftime.num2date(noleap_time1, units, calendar) - expected2 = cftime.num2date(noleap_time2, units, calendar) + expected1 = cftime.num2date(time1, units, calendar) + expected2 = cftime.num2date(time2, units, calendar) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype('O') @@ -462,66 +412,51 @@ def test_decode_multidim_time_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(['360_day', 'all_leap', '366_day'], [False, True])) -def test_decode_non_standard_calendar_single_element_fallback( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +@pytest.mark.parametrize('calendar', ['360_day', 'all_leap', '366_day']) +def test_decode_non_standard_calendar_single_element( + calendar): cftime = _import_cftime() - units = 'days since 0001-01-01' + try: dt = cftime.netcdftime.datetime(2001, 2, 29) except AttributeError: - # Must be using standalone netcdftime library + # Must be using the standalone cftime library dt = cftime.datetime(2001, 2, 29) num_time = cftime.date2num(dt, units, calendar) - if enable_cftimeindex: - actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - else: - with pytest.warns(SerializationWarning, - match='Unable to decode time axis'): - actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar) - expected = np.asarray(cftime.num2date(num_time, units, calendar)) + if cftime.__name__ == 'cftime': + expected = np.asarray(cftime.num2date( + num_time, units, calendar, only_use_cftime_datetimes=True)) + else: + expected = np.asarray(cftime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(['360_day'], [False, True])) -def test_decode_non_standard_calendar_fallback( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +def test_decode_360_day_calendar(): cftime = _import_cftime() + calendar = '360_day' # ensure leap year doesn't matter for year in [2010, 2011, 2012, 2013, 2014]: units = 'days since {0}-01-01'.format(year) num_times = np.arange(100) - expected = cftime.num2date(num_times, units, calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(num_times, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(num_times, units, calendar) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') actual = coding.times.decode_cf_datetime( - num_times, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - if enable_cftimeindex: - assert len(w) == 0 - else: - assert len(w) == 1 - assert 'Unable to decode time axis' in str(w[0].message) + num_times, units, calendar=calendar) + assert len(w) == 0 assert actual.dtype == np.dtype('O') assert_array_equal(actual, expected) @@ -667,11 +602,8 @@ def test_format_cftime_datetime(date_args, expected): assert result == expected -@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) -def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_decode_cf(calendar): days = [1., 2., 3.] da = DataArray(days, coords=[days], dims=['time'], name='test') ds = da.to_dataset() @@ -680,17 +612,13 @@ def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): ds[v].attrs['units'] = 'days since 2001-01-01' ds[v].attrs['calendar'] = calendar - if (not has_cftime and enable_cftimeindex and - calendar not in _STANDARD_CALENDARS): + if not has_cftime_or_netCDF4 and calendar not in _STANDARD_CALENDARS: with pytest.raises(ValueError): - with set_options(enable_cftimeindex=enable_cftimeindex): - ds = decode_cf(ds) - else: - with set_options(enable_cftimeindex=enable_cftimeindex): ds = decode_cf(ds) + else: + ds = decode_cf(ds) - if (enable_cftimeindex and - calendar not in _STANDARD_CALENDARS): + if calendar not in _STANDARD_CALENDARS: assert ds.test.dtype == np.dtype('O') else: assert ds.test.dtype == np.dtype('M8[ns]') @@ -764,7 +692,7 @@ def test_contains_cftime_datetimes_non_cftimes_dask(non_cftime_data): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') @pytest.mark.parametrize('shape', [(24,), (8, 3), (2, 4, 3)]) -def test_encode_datetime_overflow(shape): +def test_encode_cf_datetime_overflow(shape): # Test for fix to GH 2272 dates = pd.date_range('2100', periods=24).values.reshape(shape) units = 'days since 1800-01-01' diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 433a669e340..abdbdbdbfdf 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2279,12 +2279,9 @@ def test_resample_cftimeindex(self): cftime = _import_cftime() times = cftime.num2date(np.arange(12), units='hours since 0001-01-01', calendar='noleap') - with set_options(enable_cftimeindex=True): - array = DataArray(np.arange(12), [('time', times)]) + array = DataArray(np.arange(12), [('time', times)]) - with raises_regex(TypeError, - 'Only valid with DatetimeIndex, ' - 'TimedeltaIndex or PeriodIndex'): + with raises_regex(NotImplementedError, 'to_datetimeindex'): array.resample(time='6H').mean() def test_resample_first(self): diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index a21ea3e6b64..d594e1dcd18 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -33,8 +33,9 @@ def test_arithmetic_join(): def test_enable_cftimeindex(): with pytest.raises(ValueError): xarray.set_options(enable_cftimeindex=None) - with xarray.set_options(enable_cftimeindex=True): - assert OPTIONS['enable_cftimeindex'] + with pytest.warns(FutureWarning, match='no-op'): + with xarray.set_options(enable_cftimeindex=True): + assert OPTIONS['enable_cftimeindex'] def test_file_cache_maxsize(): diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 33021fc5ef4..ed07af0d7bb 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -46,19 +46,17 @@ def test_safe_cast_to_index(): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_safe_cast_to_index_cftimeindex(enable_cftimeindex): +def test_safe_cast_to_index_cftimeindex(): date_types = _all_cftime_date_types() for date_type in date_types.values(): dates = [date_type(1, 1, day) for day in range(1, 20)] - if enable_cftimeindex and has_cftime: + if has_cftime: expected = CFTimeIndex(dates) else: expected = pd.Index(dates) - with set_options(enable_cftimeindex=enable_cftimeindex): - actual = utils.safe_cast_to_index(np.array(dates)) + actual = utils.safe_cast_to_index(np.array(dates)) assert_array_equal(expected, actual) assert expected.dtype == actual.dtype assert isinstance(actual, type(expected)) @@ -66,13 +64,11 @@ def test_safe_cast_to_index_cftimeindex(enable_cftimeindex): # Test that datetime.datetime objects are never used in a CFTimeIndex @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_safe_cast_to_index_datetime_datetime(enable_cftimeindex): +def test_safe_cast_to_index_datetime_datetime(): dates = [datetime(1, 1, day) for day in range(1, 20)] expected = pd.Index(dates) - with set_options(enable_cftimeindex=enable_cftimeindex): - actual = utils.safe_cast_to_index(np.array(dates)) + actual = utils.safe_cast_to_index(np.array(dates)) assert_array_equal(expected, actual) assert isinstance(actual, pd.Index) From cf798c5eb1b3c3d5f487420861cd4498f6614283 Mon Sep 17 00:00:00 2001 From: David Wang <44521652+DavidloveEllie@users.noreply.github.com> Date: Fri, 2 Nov 2018 12:36:56 +0800 Subject: [PATCH 270/282] Include multidimensional stacking groupby in docs (#2493) (#2536) --- doc/groupby.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/groupby.rst b/doc/groupby.rst index 4851cbe5dcc..6e42dbbc9f0 100644 --- a/doc/groupby.rst +++ b/doc/groupby.rst @@ -207,3 +207,12 @@ may be desirable: .. ipython:: python da.groupby_bins('lon', [0,45,50]).sum() + +These methods group by `lon` values. It is also possible to groupby each +cell in a grid, regardless of value, by stacking multiple dimensions, +applying your function, and then unstacking the result: + +.. ipython:: python + + stacked = da.stack(gridcell=['ny', 'nx']) + stacked.groupby('gridcell').sum().unstack('gridcell') From f788084f6672e1325938ba2f6a0bd105aa412091 Mon Sep 17 00:00:00 2001 From: Lily Wang <31115101+lilyminium@users.noreply.github.com> Date: Fri, 2 Nov 2018 15:59:03 +1100 Subject: [PATCH 271/282] Zarr chunking (GH2300) (#2487) * fixed typo * added test for saving opened zarr dataset * modified test for saving opened zarr dataset * allow different last chunk * removed whitespace * modified error messages * fixed pep8 issues * updated whats-new --- doc/whats-new.rst | 4 ++++ xarray/backends/zarr.py | 19 +++++++++++++------ xarray/tests/test_backends.py | 4 ++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 9db3d35af84..61d6eda4333 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -152,6 +152,10 @@ Bug fixes the dates must be encoded using cftime rather than NumPy (:issue:`2272`). By `Spencer Clark `_. +- Chunked datasets can now roundtrip to Zarr storage continually + with `to_zarr` and ``open_zarr`` (:issue:`2300`). + By `Lily Wang `_. + .. _whats-new.0.10.9: v0.10.9 (21 September 2018) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 5f19c826289..06fe7f04e4f 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -79,14 +79,14 @@ def _determine_zarr_chunks(enc_chunks, var_chunks, ndim): if var_chunks and enc_chunks is None: if any(len(set(chunks[:-1])) > 1 for chunks in var_chunks): raise ValueError( - "Zarr requires uniform chunk sizes excpet for final chunk." - " Variable %r has incompatible chunks. Consider " + "Zarr requires uniform chunk sizes except for final chunk." + " Variable dask chunks %r are incompatible. Consider " "rechunking using `chunk()`." % (var_chunks,)) if any((chunks[0] < chunks[-1]) for chunks in var_chunks): raise ValueError( - "Final chunk of Zarr array must be smaller than first. " - "Variable %r has incompatible chunks. Consider rechunking " - "using `chunk()`." % var_chunks) + "Final chunk of Zarr array must be the same size or smaller " + "than the first. Variable Dask chunks %r are incompatible. " + "Consider rechunking using `chunk()`." % var_chunks) # return the first chunk for each dimension return tuple(chunk[0] for chunk in var_chunks) @@ -126,7 +126,7 @@ def _determine_zarr_chunks(enc_chunks, var_chunks, ndim): # threads if var_chunks and enc_chunks_tuple: for zchunk, dchunks in zip(enc_chunks_tuple, var_chunks): - for dchunk in dchunks: + for dchunk in dchunks[:-1]: if dchunk % zchunk: raise NotImplementedError( "Specified zarr chunks %r would overlap multiple dask " @@ -134,6 +134,13 @@ def _determine_zarr_chunks(enc_chunks, var_chunks, ndim): " Consider rechunking the data using " "`chunk()` or specifying different chunks in encoding." % (enc_chunks_tuple, var_chunks)) + if dchunks[-1] > zchunk: + raise ValueError( + "Final chunk of Zarr array must be the same size or " + "smaller than the first. The specified Zarr chunk " + "encoding is %r, but %r in variable Dask chunks %r is " + "incompatible. Consider rechunking using `chunk()`." + % (enc_chunks_tuple, dchunks, var_chunks)) return enc_chunks_tuple raise AssertionError( diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 80d3a9d526e..fb9c43c0165 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1388,6 +1388,10 @@ def test_chunk_encoding_with_dask(self): ds_chunk_irreg = ds.chunk({'x': (5, 5, 2)}) with self.roundtrip(ds_chunk_irreg) as actual: assert (5,) == actual['var1'].encoding['chunks'] + # re-save Zarr arrays + with self.roundtrip(ds_chunk_irreg) as original: + with self.roundtrip(original) as actual: + assert_identical(original, actual) # - encoding specified - # specify compatible encodings From 848d491826a746711265b42a12fec12611fe539a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sat, 3 Nov 2018 14:24:12 -0700 Subject: [PATCH 272/282] Deprecate inplace (#2524) * Start deprecating inplace. * remove warnings from tests. * this commit silences nearly all warnings. * Add whats-new. * Add a default kwarg to _check_inplace and use for Dataset.update. * Major fix! * Add stacklevel * Tests: Less aggressive warning filter + fix unnecessary inplace. * revert changes to _calculate_binary_op --- doc/whats-new.rst | 3 +++ xarray/core/dataarray.py | 15 ++++++++++----- xarray/core/dataset.py | 34 +++++++++++++++++++++------------- xarray/core/utils.py | 10 ++++++++++ xarray/tests/test_dataarray.py | 18 ++++++++++-------- xarray/tests/test_dataset.py | 33 +++++++++++++++++++-------------- xarray/tests/test_plot.py | 2 +- 7 files changed, 74 insertions(+), 41 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 61d6eda4333..8632d97be4b 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -77,6 +77,9 @@ Documentation without dimension argument will change in the next release. Now we warn a FutureWarning. By `Keisuke Fujii `_. +- The ``inplace`` kwarg of a number of `DataArray` and `Dataset` methods is being + deprecated and will be removed in the next release. + By `Deepak Cherian `_. Enhancements ~~~~~~~~~~~~ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index bccfd8b79d4..17af3cf2cd1 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -19,7 +19,8 @@ from .options import OPTIONS, _get_keep_attrs from .pycompat import OrderedDict, basestring, iteritems, range, zip from .utils import ( - decode_numpy_dict_values, either_dict_or_kwargs, ensure_us_time_resolution) + _check_inplace, decode_numpy_dict_values, either_dict_or_kwargs, + ensure_us_time_resolution) from .variable import ( IndexVariable, Variable, as_compatible_data, as_variable, assert_unique_multiindex_level_names) @@ -542,7 +543,7 @@ def coords(self): """ return DataArrayCoordinates(self) - def reset_coords(self, names=None, drop=False, inplace=False): + def reset_coords(self, names=None, drop=False, inplace=None): """Given names of coordinates, reset them to become variables. Parameters @@ -561,6 +562,7 @@ def reset_coords(self, names=None, drop=False, inplace=False): ------- Dataset, or DataArray if ``drop == True`` """ + inplace = _check_inplace(inplace) if inplace and not drop: raise ValueError('cannot reset coordinates in-place on a ' 'DataArray without ``drop == True``') @@ -1151,7 +1153,7 @@ def expand_dims(self, dim, axis=None): ds = self._to_temp_dataset().expand_dims(dim, axis) return self._from_temp_dataset(ds) - def set_index(self, indexes=None, append=False, inplace=False, + def set_index(self, indexes=None, append=False, inplace=None, **indexes_kwargs): """Set DataArray (multi-)indexes using one or more existing coordinates. @@ -1181,6 +1183,7 @@ def set_index(self, indexes=None, append=False, inplace=False, -------- DataArray.reset_index """ + inplace = _check_inplace(inplace) indexes = either_dict_or_kwargs(indexes, indexes_kwargs, 'set_index') coords, _ = merge_indexes(indexes, self._coords, set(), append=append) if inplace: @@ -1188,7 +1191,7 @@ def set_index(self, indexes=None, append=False, inplace=False, else: return self._replace(coords=coords) - def reset_index(self, dims_or_levels, drop=False, inplace=False): + def reset_index(self, dims_or_levels, drop=False, inplace=None): """Reset the specified index(es) or multi-index level(s). Parameters @@ -1213,6 +1216,7 @@ def reset_index(self, dims_or_levels, drop=False, inplace=False): -------- DataArray.set_index """ + inplace = _check_inplace(inplace) coords, _ = split_indexes(dims_or_levels, self._coords, set(), self._level_coords, drop=drop) if inplace: @@ -1220,7 +1224,7 @@ def reset_index(self, dims_or_levels, drop=False, inplace=False): else: return self._replace(coords=coords) - def reorder_levels(self, dim_order=None, inplace=False, + def reorder_levels(self, dim_order=None, inplace=None, **dim_order_kwargs): """Rearrange index levels using input order. @@ -1243,6 +1247,7 @@ def reorder_levels(self, dim_order=None, inplace=False, Another dataarray, with this dataarray's data but replaced coordinates. """ + inplace = _check_inplace(inplace) dim_order = either_dict_or_kwargs(dim_order, dim_order_kwargs, 'reorder_levels') replace_coords = {} diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 7bd99968ebb..4f9c61b3269 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -32,9 +32,9 @@ from .pycompat import ( OrderedDict, basestring, dask_array_type, integer_types, iteritems, range) from .utils import ( - Frozen, SortedKeysDict, datetime_to_numeric, decode_numpy_dict_values, - either_dict_or_kwargs, ensure_us_time_resolution, hashable, - maybe_wrap_array) + _check_inplace, Frozen, SortedKeysDict, datetime_to_numeric, + decode_numpy_dict_values, either_dict_or_kwargs, ensure_us_time_resolution, + hashable, maybe_wrap_array) from .variable import IndexVariable, Variable, as_variable, broadcast_variables # list of attributes of pd.DatetimeIndex that are ndarrays of time info @@ -1072,7 +1072,7 @@ def data_vars(self): """ return DataVariables(self) - def set_coords(self, names, inplace=False): + def set_coords(self, names, inplace=None): """Given names of one or more variables, set them as coordinates Parameters @@ -1095,6 +1095,7 @@ def set_coords(self, names, inplace=False): # DataFrame.set_index? # nb. check in self._variables, not self.data_vars to insure that the # operation is idempotent + inplace = _check_inplace(inplace) if isinstance(names, basestring): names = [names] self._assert_all_in_dataset(names) @@ -1102,7 +1103,7 @@ def set_coords(self, names, inplace=False): obj._coord_names.update(names) return obj - def reset_coords(self, names=None, drop=False, inplace=False): + def reset_coords(self, names=None, drop=False, inplace=None): """Given names of coordinates, reset them to become variables Parameters @@ -1121,6 +1122,7 @@ def reset_coords(self, names=None, drop=False, inplace=False): ------- Dataset """ + inplace = _check_inplace(inplace) if names is None: names = self._coord_names - set(self.dims) else: @@ -2045,7 +2047,7 @@ def interp_like(self, other, method='linear', assume_sorted=False, ds = self.reindex(object_coords) return ds.interp(numeric_coords, method, assume_sorted, kwargs) - def rename(self, name_dict=None, inplace=False, **names): + def rename(self, name_dict=None, inplace=None, **names): """Returns a new object with renamed variables and dimensions. Parameters @@ -2070,6 +2072,7 @@ def rename(self, name_dict=None, inplace=False, **names): Dataset.swap_dims DataArray.rename """ + inplace = _check_inplace(inplace) name_dict = either_dict_or_kwargs(name_dict, names, 'rename') for k, v in name_dict.items(): if k not in self and k not in self.dims: @@ -2095,7 +2098,7 @@ def rename(self, name_dict=None, inplace=False, **names): return self._replace_vars_and_dims(variables, coord_names, dims=dims, inplace=inplace) - def swap_dims(self, dims_dict, inplace=False): + def swap_dims(self, dims_dict, inplace=None): """Returns a new object with swapped dimensions. Parameters @@ -2119,6 +2122,7 @@ def swap_dims(self, dims_dict, inplace=False): Dataset.rename DataArray.swap_dims """ + inplace = _check_inplace(inplace) for k, v in dims_dict.items(): if k not in self.dims: raise ValueError('cannot swap from dimension %r because it is ' @@ -2231,7 +2235,7 @@ def expand_dims(self, dim, axis=None): return self._replace_vars_and_dims(variables, self._coord_names) - def set_index(self, indexes=None, append=False, inplace=False, + def set_index(self, indexes=None, append=False, inplace=None, **indexes_kwargs): """Set Dataset (multi-)indexes using one or more existing coordinates or variables. @@ -2262,6 +2266,7 @@ def set_index(self, indexes=None, append=False, inplace=False, Dataset.reset_index Dataset.swap_dims """ + inplace = _check_inplace(inplace) indexes = either_dict_or_kwargs(indexes, indexes_kwargs, 'set_index') variables, coord_names = merge_indexes(indexes, self._variables, self._coord_names, @@ -2269,7 +2274,7 @@ def set_index(self, indexes=None, append=False, inplace=False, return self._replace_vars_and_dims(variables, coord_names=coord_names, inplace=inplace) - def reset_index(self, dims_or_levels, drop=False, inplace=False): + def reset_index(self, dims_or_levels, drop=False, inplace=None): """Reset the specified index(es) or multi-index level(s). Parameters @@ -2293,13 +2298,14 @@ def reset_index(self, dims_or_levels, drop=False, inplace=False): -------- Dataset.set_index """ + inplace = _check_inplace(inplace) variables, coord_names = split_indexes(dims_or_levels, self._variables, self._coord_names, self._level_coords, drop=drop) return self._replace_vars_and_dims(variables, coord_names=coord_names, inplace=inplace) - def reorder_levels(self, dim_order=None, inplace=False, + def reorder_levels(self, dim_order=None, inplace=None, **dim_order_kwargs): """Rearrange index levels using input order. @@ -2322,6 +2328,7 @@ def reorder_levels(self, dim_order=None, inplace=False, Another dataset, with this dataset's data but replaced coordinates. """ + inplace = _check_inplace(inplace) dim_order = either_dict_or_kwargs(dim_order, dim_order_kwargs, 'reorder_levels') replace_variables = {} @@ -2472,7 +2479,7 @@ def unstack(self, dim=None): result = result._unstack_once(dim) return result - def update(self, other, inplace=True): + def update(self, other, inplace=None): """Update this dataset's variables with those from another dataset. Parameters @@ -2494,12 +2501,13 @@ def update(self, other, inplace=True): If any dimensions would have inconsistent sizes in the updated dataset. """ + inplace = _check_inplace(inplace, default=True) variables, coord_names, dims = dataset_update_method(self, other) return self._replace_vars_and_dims(variables, coord_names, dims, inplace=inplace) - def merge(self, other, inplace=False, overwrite_vars=frozenset(), + def merge(self, other, inplace=None, overwrite_vars=frozenset(), compat='no_conflicts', join='outer'): """Merge the arrays of two datasets into a single dataset. @@ -2550,6 +2558,7 @@ def merge(self, other, inplace=False, overwrite_vars=frozenset(), MergeError If any variables conflict (see ``compat``). """ + inplace = _check_inplace(inplace) variables, coord_names, dims = dataset_merge_method( self, other, overwrite_vars=overwrite_vars, compat=compat, join=join) @@ -3317,7 +3326,6 @@ def func(self, other): def _calculate_binary_op(self, f, other, join='inner', inplace=False): - def apply_over_both(lhs_data_vars, rhs_data_vars, lhs_vars, rhs_vars): if inplace and set(lhs_data_vars) != set(rhs_data_vars): raise ValueError('datasets must have the same data variables ' diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 015916d668e..50d6ec7e05a 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -17,6 +17,16 @@ OrderedDict, basestring, bytes_type, dask_array_type, iteritems) +def _check_inplace(inplace, default=False): + if inplace is None: + inplace = default + else: + warnings.warn('The inplace argument has been deprecated and will be ' + 'removed in xarray 0.12.0.', FutureWarning, stacklevel=3) + + return inplace + + def alias_message(old_name, new_name): return '%s has been deprecated. Use %s instead.' % (old_name, new_name) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index abdbdbdbfdf..2b35921fae8 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1155,7 +1155,7 @@ def test_reset_coords(self): assert_identical(actual, expected) actual = data.copy() - actual.reset_coords(drop=True, inplace=True) + actual = actual.reset_coords(drop=True) assert_identical(actual, expected) actual = data.reset_coords('bar', drop=True) @@ -1164,8 +1164,9 @@ def test_reset_coords(self): dims=['x', 'y'], name='foo') assert_identical(actual, expected) - with raises_regex(ValueError, 'cannot reset coord'): - data.reset_coords(inplace=True) + with pytest.warns(FutureWarning, message='The inplace argument'): + with raises_regex(ValueError, 'cannot reset coord'): + data = data.reset_coords(inplace=True) with raises_regex(ValueError, 'cannot be found'): data.reset_coords('foo', drop=True) with raises_regex(ValueError, 'cannot be found'): @@ -1398,7 +1399,7 @@ def test_set_index(self): expected = array.set_index(x=['level_1', 'level_2', 'level_3']) assert_identical(obj, expected) - array.set_index(x=['level_1', 'level_2', 'level_3'], inplace=True) + array = array.set_index(x=['level_1', 'level_2', 'level_3']) assert_identical(array, expected) array2d = DataArray(np.random.rand(2, 2), @@ -1431,7 +1432,7 @@ def test_reset_index(self): assert_identical(obj, expected) array = self.mda.copy() - array.reset_index(['x'], drop=True, inplace=True) + array = array.reset_index(['x'], drop=True) assert_identical(array, expected) # single index @@ -1447,9 +1448,10 @@ def test_reorder_levels(self): obj = self.mda.reorder_levels(x=['level_2', 'level_1']) assert_identical(obj, expected) - array = self.mda.copy() - array.reorder_levels(x=['level_2', 'level_1'], inplace=True) - assert_identical(array, expected) + with pytest.warns(FutureWarning, message='The inplace argument'): + array = self.mda.copy() + array.reorder_levels(x=['level_2', 'level_1'], inplace=True) + assert_identical(array, expected) array = DataArray([1, 2], dims='x') with pytest.raises(KeyError): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index a32253c19e5..8c0f8508df9 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1965,6 +1965,7 @@ def test_rename_same_name(self): renamed = data.rename(newnames) assert_identical(renamed, data) + @pytest.mark.filterwarnings('ignore:The inplace argument') def test_rename_inplace(self): times = pd.date_range('2000-01-01', periods=3) data = Dataset({'z': ('x', [2, 3, 4]), 't': ('t', times)}) @@ -1990,7 +1991,7 @@ def test_swap_dims(self): assert_identical(original.set_coords('y'), roundtripped) actual = original.copy() - actual.swap_dims({'x': 'y'}, inplace=True) + actual = actual.swap_dims({'x': 'y'}) assert_identical(expected, actual) with raises_regex(ValueError, 'cannot swap'): @@ -2012,7 +2013,7 @@ def test_expand_dims_error(self): # Make sure it raises true error also for non-dimensional coordinates # which has dimension. - original.set_coords('z', inplace=True) + original = original.set_coords('z') with raises_regex(ValueError, 'already exists'): original.expand_dims(dim=['z']) @@ -2059,8 +2060,9 @@ def test_set_index(self): obj = ds.set_index(x=mindex.names) assert_identical(obj, expected) - ds.set_index(x=mindex.names, inplace=True) - assert_identical(ds, expected) + with pytest.warns(FutureWarning, message='The inplace argument'): + ds.set_index(x=mindex.names, inplace=True) + assert_identical(ds, expected) # ensure set_index with no existing index and a single data var given # doesn't return multi-index @@ -2078,8 +2080,9 @@ def test_reset_index(self): obj = ds.reset_index('x') assert_identical(obj, expected) - ds.reset_index('x', inplace=True) - assert_identical(ds, expected) + with pytest.warns(FutureWarning, message='The inplace argument'): + ds.reset_index('x', inplace=True) + assert_identical(ds, expected) def test_reorder_levels(self): ds = create_test_multiindex() @@ -2090,8 +2093,9 @@ def test_reorder_levels(self): reindexed = ds.reorder_levels(x=['level_2', 'level_1']) assert_identical(reindexed, expected) - ds.reorder_levels(x=['level_2', 'level_1'], inplace=True) - assert_identical(ds, expected) + with pytest.warns(FutureWarning, message='The inplace argument'): + ds.reorder_levels(x=['level_2', 'level_1'], inplace=True) + assert_identical(ds, expected) ds = Dataset({}, coords={'x': [1, 2]}) with raises_regex(ValueError, 'has no MultiIndex'): @@ -2169,14 +2173,15 @@ def test_update(self): assert_identical(expected, actual) actual = data.copy() - actual_result = actual.update(data, inplace=True) + actual_result = actual.update(data) assert actual_result is actual assert_identical(expected, actual) - actual = data.update(data, inplace=False) - expected = data - assert actual is not expected - assert_identical(expected, actual) + with pytest.warns(FutureWarning, message='The inplace argument'): + actual = data.update(data, inplace=False) + expected = data + assert actual is not expected + assert_identical(expected, actual) other = Dataset(attrs={'new': 'attr'}) actual = data.copy() @@ -2556,7 +2561,7 @@ def get_args(v): return [set(args[0]) & set(v.dims)] if args else [] expected = Dataset(dict((k, v.squeeze(*get_args(v))) for k, v in iteritems(data.variables))) - expected.set_coords(data.coords, inplace=True) + expected = expected.set_coords(data.coords) assert_identical(expected, data.squeeze(*args)) # invalid squeeze with raises_regex(ValueError, 'cannot select a dimension'): diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 38cf68e47cc..10c4283032d 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -797,7 +797,7 @@ def setUp(self): x, y = np.meshgrid(da.x.values, da.y.values) ds['x2d'] = DataArray(x, dims=['y', 'x']) ds['y2d'] = DataArray(y, dims=['y', 'x']) - ds.set_coords(['x2d', 'y2d'], inplace=True) + ds = ds.set_coords(['x2d', 'y2d']) # set darray and plot method self.darray = ds.testvar From 38399cce9b97a7b48d56f66500469305ccc250f9 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Mon, 5 Nov 2018 03:35:27 +0100 Subject: [PATCH 273/282] Remove use of deprecated, unused keyword. (#2540) --- xarray/backends/cfgrib_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index c0a7c025606..37d86d8a427 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -46,7 +46,7 @@ def __init__(self, filename, lock=None, **backend_kwargs): filter_by_keys_items = backend_kwargs['filter_by_keys'].items() backend_kwargs['filter_by_keys'] = tuple(filter_by_keys_items) - self.ds = cfgrib.open_file(filename, mode='r', **backend_kwargs) + self.ds = cfgrib.open_file(filename, **backend_kwargs) def open_store_variable(self, name, var): if isinstance(var.data, np.ndarray): From 421be442041e6dbaa47934cb223cb28dd2b37e53 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 4 Nov 2018 20:27:06 -0800 Subject: [PATCH 274/282] Remove the old syntax for resample. (#2541) This has been deprecated since xarray 0.10. I also added support for passing a mapping ``{dim: freq}`` as the first argument. --- doc/whats-new.rst | 9 ++++ xarray/core/common.py | 94 ++++++++-------------------------- xarray/tests/test_dataarray.py | 53 +++++-------------- xarray/tests/test_dataset.py | 19 ++++--- 4 files changed, 51 insertions(+), 124 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8632d97be4b..2347d880350 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -57,6 +57,11 @@ Breaking changes includes only data variables. - ``DataArray.__contains__`` (used by Python's ``in`` operator) now checks array data, not coordinates. + - The old resample syntax from before xarray 0.10, e.g., + ``data.resample('1D', dim='time', how='mean')``, is no longer supported will + raise an error in most cases. You need to use the new resample syntax + instead, e.g., ``data.resample(time='1D').mean()`` or + ``data.resample({'time': '1D'}).mean()``. - Xarray's storage backends now automatically open and close files when necessary, rather than requiring opening a file with ``autoclose=True``. A global least-recently-used cache is used to store open files; the default @@ -111,6 +116,10 @@ Enhancements python driver and *ecCodes* C-library. (:issue:`2475`) By `Alessandro Amici `_, sponsored by `ECMWF `_. +- Resample now supports a dictionary mapping from dimension to frequency as + its first argument, e.g., ``data.resample({'time': '1D'}).mean()``. This is + consistent with other xarray functions that accept either dictionaries or + keyword arguments. By `Stephan Hoyer `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/core/common.py b/xarray/core/common.py index 508a19b7115..34057e3715d 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -548,7 +548,7 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): Set the labels at the center of the window. **dim_kwargs : optional The keyword arguments form of ``dim``. - One of dim or dim_kwarg must be provided. + One of dim or dim_kwargs must be provided. Returns ------- @@ -591,8 +591,8 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): return self._rolling_cls(self, dim, min_periods=min_periods, center=center) - def resample(self, freq=None, dim=None, how=None, skipna=None, - closed=None, label=None, base=0, keep_attrs=None, **indexer): + def resample(self, indexer=None, skipna=None, closed=None, label=None, + base=0, keep_attrs=None, **indexer_kwargs): """Returns a Resample object for performing resampling operations. Handles both downsampling and upsampling. If any intervals contain no @@ -600,6 +600,8 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, Parameters ---------- + indexer : {dim: freq}, optional + Mapping from the dimension name to resample frequency. skipna : bool, optional Whether to skip missing values when aggregating in downsampling. closed : 'left' or 'right', optional @@ -614,9 +616,9 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, If True, the object's attributes (`attrs`) will be copied from the original object to the new one. If False (default), the new object will be returned without attributes. - **indexer : {dim: freq} - Dictionary with a key indicating the dimension name to resample - over and a value corresponding to the resampling frequency. + **indexer_kwargs : {dim: freq} + The keyword arguments form of ``indexer``. + One of indexer or indexer_kwargs must be provided. Returns ------- @@ -664,30 +666,24 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, if keep_attrs is None: keep_attrs = _get_keep_attrs(default=False) - if dim is not None: - if how is None: - how = 'mean' - return self._resample_immediately(freq, dim, how, skipna, closed, - label, base, keep_attrs) + # note: the second argument (now 'skipna') use to be 'dim' + if ((skipna is not None and not isinstance(skipna, bool)) + or ('how' in indexer_kwargs and 'how' not in self.dims) + or ('dim' in indexer_kwargs and 'dim' not in self.dims)): + raise TypeError('resample() no longer supports the `how` or ' + '`dim` arguments. Instead call methods on resample ' + "objects, e.g., data.resample(time='1D').mean()") + + indexer = either_dict_or_kwargs(indexer, indexer_kwargs, 'resample') - if (how is not None) and indexer: - raise TypeError("If passing an 'indexer' then 'dim' " - "and 'how' should not be used") - - # More than one indexer is ambiguous, but we do in fact need one if - # "dim" was not provided, until the old API is fully deprecated if len(indexer) != 1: raise ValueError( "Resampling only supported along single dimensions." ) dim, freq = indexer.popitem() - if isinstance(dim, basestring): - dim_name = dim - dim = self[dim] - else: - raise TypeError("Dimension name should be a string; " - "was passed %r" % dim) + dim_name = dim + dim_coord = self[dim] if isinstance(self.indexes[dim_name], CFTimeIndex): raise NotImplementedError( @@ -702,7 +698,8 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, 'errors.' ) - group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) + group = DataArray(dim_coord, coords=dim_coord.coords, + dims=dim_coord.dims, name=RESAMPLE_DIM) grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base) resampler = self._resample_cls(self, group=group, dim=dim_name, grouper=grouper, @@ -710,55 +707,6 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, return resampler - def _resample_immediately(self, freq, dim, how, skipna, - closed, label, base, keep_attrs): - """Implement the original version of .resample() which immediately - executes the desired resampling operation. """ - from .dataarray import DataArray - from ..coding.cftimeindex import CFTimeIndex - - RESAMPLE_DIM = '__resample_dim__' - - warnings.warn("\n.resample() has been modified to defer " - "calculations. Instead of passing 'dim' and " - "how=\"{how}\", instead consider using " - ".resample({dim}=\"{freq}\").{how}('{dim}') ".format( - dim=dim, freq=freq, how=how), - FutureWarning, stacklevel=3) - - if isinstance(self.indexes[dim], CFTimeIndex): - raise NotImplementedError( - 'Resample is currently not supported along a dimension ' - 'indexed by a CFTimeIndex. For certain kinds of downsampling ' - 'it may be possible to work around this by converting your ' - 'time index to a DatetimeIndex using ' - 'CFTimeIndex.to_datetimeindex. Use caution when doing this ' - 'however, because switching to a DatetimeIndex from a ' - 'CFTimeIndex with a non-standard calendar entails a change ' - 'in the calendar type, which could lead to subtle and silent ' - 'errors.' - ) - - if isinstance(dim, basestring): - dim = self[dim] - - group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) - grouper = pd.Grouper(freq=freq, how=how, closed=closed, label=label, - base=base) - gb = self._groupby_cls(self, group, grouper=grouper) - if isinstance(how, basestring): - f = getattr(gb, how) - if how in ['first', 'last']: - result = f(skipna=skipna, keep_attrs=keep_attrs) - elif how == 'count': - result = f(dim=dim.name, keep_attrs=keep_attrs) - else: - result = f(dim=dim.name, skipna=skipna, keep_attrs=keep_attrs) - else: - result = gb.reduce(how, dim=dim.name, keep_attrs=keep_attrs) - result = result.rename({RESAMPLE_DIM: dim.name}) - return result - def where(self, cond, other=dtypes.NA, drop=False): """Filter elements from this object according to a condition. diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 2b35921fae8..87ee60715a1 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2358,53 +2358,24 @@ def test_resample_drop_nondim_coords(self): actual = array.resample(time="1H").interpolate('linear') assert 'tc' not in actual.coords - def test_resample_old_vs_new_api(self): + def test_resample_keep_attrs(self): times = pd.date_range('2000-01-01', freq='6H', periods=10) array = DataArray(np.ones(10), [('time', times)]) + array.attrs['meta'] = 'data' - # Simple mean - with pytest.warns(FutureWarning): - old_mean = array.resample('1D', 'time', how='mean') - new_mean = array.resample(time='1D').mean() - assert_identical(old_mean, new_mean) - - # Mean, while keeping attributes - attr_array = array.copy() - attr_array.attrs['meta'] = 'data' - - with pytest.warns(FutureWarning): - old_mean = attr_array.resample('1D', dim='time', how='mean', - keep_attrs=True) - new_mean = attr_array.resample(time='1D').mean(keep_attrs=True) - assert old_mean.attrs == new_mean.attrs - assert_identical(old_mean, new_mean) + result = array.resample(time='1D').mean(keep_attrs=True) + expected = DataArray([1, 1, 1], [('time', times[::4])], + attrs=array.attrs) + assert_identical(result, expected) - # Mean, with NaN to skip - nan_array = array.copy() - nan_array[1] = np.nan + def test_resample_skipna(self): + times = pd.date_range('2000-01-01', freq='6H', periods=10) + array = DataArray(np.ones(10), [('time', times)]) + array[1] = np.nan - with pytest.warns(FutureWarning): - old_mean = nan_array.resample('1D', 'time', how='mean', - skipna=False) - new_mean = nan_array.resample(time='1D').mean(skipna=False) + result = array.resample(time='1D').mean(skipna=False) expected = DataArray([np.nan, 1, 1], [('time', times[::4])]) - assert_identical(old_mean, expected) - assert_identical(new_mean, expected) - - # Try other common resampling methods - resampler = array.resample(time='1D') - for method in ['mean', 'median', 'sum', 'first', 'last', 'count']: - # Discard attributes on the call using the new api to match - # convention from old api - new_api = getattr(resampler, method)(keep_attrs=False) - with pytest.warns(FutureWarning): - old_api = array.resample('1D', dim='time', how=method) - assert_identical(new_api, old_api) - for method in [np.mean, np.sum, np.max, np.min]: - new_api = resampler.reduce(method) - with pytest.warns(FutureWarning): - old_api = array.resample('1D', dim='time', how=method) - assert_identical(new_api, old_api) + assert_identical(result, expected) def test_upsample(self): times = pd.date_range('2000-01-01', freq='6H', periods=5) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 8c0f8508df9..89ea3ba78a0 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2858,22 +2858,21 @@ def test_resample_drop_nondim_coords(self): actual = ds.resample(time="1H").interpolate('linear') assert 'tc' not in actual.coords - def test_resample_old_vs_new_api(self): + def test_resample_old_api(self): times = pd.date_range('2000-01-01', freq='6H', periods=10) ds = Dataset({'foo': (['time', 'x', 'y'], np.random.randn(10, 5, 3)), 'bar': ('time', np.random.randn(10), {'meta': 'data'}), 'time': times}) - ds.attrs['dsmeta'] = 'dsdata' - for method in ['mean', 'sum', 'count', 'first', 'last']: - resampler = ds.resample(time='1D') - # Discard attributes on the call using the new api to match - # convention from old api - new_api = getattr(resampler, method)(keep_attrs=False) - with pytest.warns(FutureWarning): - old_api = ds.resample('1D', dim='time', how=method) - assert_identical(new_api, old_api) + with raises_regex(TypeError, r'resample\(\) no longer supports'): + ds.resample('1D', 'time') + + with raises_regex(TypeError, r'resample\(\) no longer supports'): + ds.resample('1D', dim='time', how='mean') + + with raises_regex(TypeError, r'resample\(\) no longer supports'): + ds.resample('1D', dim='time') def test_to_array(self): ds = Dataset(OrderedDict([('a', 1), ('b', ('x', [1, 2, 3]))]), From 55f21deff4c2b42bd6ead4dbe26a1b123337913a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 5 Nov 2018 07:36:16 -0800 Subject: [PATCH 275/282] Stop loading tutorial data by default (#2538) * putting up for discussion: stop loading tutorial data by default * add tutorial.open_dataset * fix typo * add test for cached tutoreial data and minor doc fixes --- doc/indexing.rst | 2 +- doc/interpolation.rst | 2 +- doc/plotting.rst | 4 ++-- doc/whats-new.rst | 5 +++++ xarray/tests/test_tutorial.py | 7 ++++++- xarray/tutorial.py | 27 +++++++++++++++++++++++++-- 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/doc/indexing.rst b/doc/indexing.rst index c05bf9994fc..3878d983cf6 100644 --- a/doc/indexing.rst +++ b/doc/indexing.rst @@ -411,7 +411,7 @@ can use indexing with ``.loc`` : .. ipython:: python - ds = xr.tutorial.load_dataset('air_temperature') + ds = xr.tutorial.open_dataset('air_temperature') #add an empty 2D dataarray ds['empty']= xr.full_like(ds.air.mean('time'),fill_value=0) diff --git a/doc/interpolation.rst b/doc/interpolation.rst index 10e46331d0a..71e88079676 100644 --- a/doc/interpolation.rst +++ b/doc/interpolation.rst @@ -262,7 +262,7 @@ Let's see how :py:meth:`~xarray.DataArray.interp` works on real data. .. ipython:: python # Raw data - ds = xr.tutorial.load_dataset('air_temperature').isel(time=0) + ds = xr.tutorial.open_dataset('air_temperature').isel(time=0) fig, axes = plt.subplots(ncols=2, figsize=(10, 4)) ds.air.plot(ax=axes[0]) axes[0].set_title('Raw data') diff --git a/doc/plotting.rst b/doc/plotting.rst index 95e63cbff05..f8ba82febb0 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -60,7 +60,7 @@ For these examples we'll use the North American air temperature dataset. .. ipython:: python - airtemps = xr.tutorial.load_dataset('air_temperature') + airtemps = xr.tutorial.open_dataset('air_temperature') airtemps # Convert to celsius @@ -585,7 +585,7 @@ This script will plot the air temperature on a map. .. ipython:: python import cartopy.crs as ccrs - air = xr.tutorial.load_dataset('air_temperature').air + air = xr.tutorial.open_dataset('air_temperature').air ax = plt.axes(projection=ccrs.Orthographic(-80, 35)) air.isel(time=0).plot.contourf(ax=ax, transform=ccrs.PlateCarree()); @savefig plotting_maps_cartopy.png width=100% diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 2347d880350..1302f8b671d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -75,6 +75,11 @@ Breaking changes should significantly improve performance when reading and writing netCDF files with Dask, especially when working with many files or using Dask Distributed. By `Stephan Hoyer `_ +- Tutorial data is now loaded lazily. Previous behavior of + :py:meth:`xarray.tutorial.load_dataset` would call `Dataset.load()` prior + to returning. This was changed in order to facilitate using this data with + dask. + By `Joe Hamman `_. Documentation ~~~~~~~~~~~~~ diff --git a/xarray/tests/test_tutorial.py b/xarray/tests/test_tutorial.py index 083ec5ee72f..6547311aa2f 100644 --- a/xarray/tests/test_tutorial.py +++ b/xarray/tests/test_tutorial.py @@ -23,6 +23,11 @@ def setUp(self): os.remove('{}.md5'.format(self.testfilepath)) def test_download_from_github(self): - ds = tutorial.load_dataset(self.testfile) + ds = tutorial.open_dataset(self.testfile).load() tiny = DataArray(range(5), name='tiny').to_dataset() assert_identical(ds, tiny) + + def test_download_from_github_load_without_cache(self): + ds_nocache = tutorial.open_dataset(self.testfile, cache=False).load() + ds_cache = tutorial.open_dataset(self.testfile).load() + assert_identical(ds_cache, ds_nocache) diff --git a/xarray/tutorial.py b/xarray/tutorial.py index 83a8317f42b..064eed330cc 100644 --- a/xarray/tutorial.py +++ b/xarray/tutorial.py @@ -9,6 +9,7 @@ import hashlib import os as _os +import warnings from .backends.api import open_dataset as _open_dataset from .core.pycompat import urlretrieve as _urlretrieve @@ -24,7 +25,7 @@ def file_md5_checksum(fname): # idea borrowed from Seaborn -def load_dataset(name, cache=True, cache_dir=_default_cache_dir, +def open_dataset(name, cache=True, cache_dir=_default_cache_dir, github_url='https://github.com/pydata/xarray-data', branch='master', **kws): """ @@ -48,6 +49,10 @@ def load_dataset(name, cache=True, cache_dir=_default_cache_dir, kws : dict, optional Passed to xarray.open_dataset + See Also + -------- + xarray.open_dataset + """ longdir = _os.path.expanduser(cache_dir) fullname = name + '.nc' @@ -77,9 +82,27 @@ def load_dataset(name, cache=True, cache_dir=_default_cache_dir, """ raise IOError(msg) - ds = _open_dataset(localfile, **kws).load() + ds = _open_dataset(localfile, **kws) if not cache: + ds = ds.load() _os.remove(localfile) return ds + + +def load_dataset(*args, **kwargs): + """ + `load_dataset` will be removed in version 0.12. The current behavior of + this function can be achived by using `tutorial.open_dataset(...).load()`. + + See Also + -------- + open_dataset + """ + warnings.warn( + "load_dataset` will be removed in xarray version 0.12. The current " + "behavior of this function can be achived by using " + "`tutorial.open_dataset(...).load()`.", + DeprecationWarning, stacklevel=2) + return open_dataset(*args, **kwargs).load() From 70f3b1cb251798335099ccdcca27ac85c70e6449 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 5 Nov 2018 11:46:29 -0500 Subject: [PATCH 276/282] Remove old-style resample example in documentation (#2543) --- doc/time-series.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index 7befd954f35..c225c246a8c 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -199,17 +199,6 @@ and ``interpolate``. ``interpolate`` extends ``scipy.interpolate.interp1d`` and supports all of its schemes. All of these resampling operations work on both Dataset and DataArray objects with an arbitrary number of dimensions. -.. note:: - - The ``resample`` api was updated in version 0.10.0 to reflect similar - updates in pandas ``resample`` api to be more groupby-like. Older style - calls to ``resample`` will still be supported for a short period: - - .. ipython:: python - - ds.resample('6H', dim='time', how='mean') - - For more examples of using grouped operations on a time dimension, see :ref:`toy weather data`. From ee5983a9d1a7389007459b8fd58c1532dd95a71d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 6 Nov 2018 08:22:45 -0800 Subject: [PATCH 277/282] add full test env for py37 ci env (#2545) --- ci/requirements-py37.yml | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 5f973936f63..6292c4c5eb6 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -1,13 +1,30 @@ name: test_env channels: - - defaults + - conda-forge dependencies: - python=3.7 + - cftime + - dask + - distributed + - h5py + - h5netcdf + - matplotlib + - netcdf4 + - pytest + - flake8 + - numpy + - pandas + - scipy + - seaborn + - toolz + - rasterio + - bottleneck + - zarr + - pseudonetcdf>=3.0.1 + - eccodes - pip: - - pytest - - flake8 - - mock - - numpy - - pandas - coveralls - pytest-cov + - pydap + - lxml + - cfgrib>=0.9.2 \ No newline at end of file From eece515aaa1bd9ec78dffe47d8d839cd69515b56 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Tue, 6 Nov 2018 17:23:16 +0100 Subject: [PATCH 278/282] Drop the hack needed to use CachingFileManager as we don't use it anymore. (#2544) --- xarray/backends/cfgrib_.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 37d86d8a427..0807900054a 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -39,13 +39,6 @@ def __init__(self, filename, lock=None, **backend_kwargs): if lock is None: lock = ECCODES_LOCK self.lock = ensure_lock(lock) - - # NOTE: filter_by_keys is a dict, but CachingFileManager only accepts - # hashable types. - if 'filter_by_keys' in backend_kwargs: - filter_by_keys_items = backend_kwargs['filter_by_keys'].items() - backend_kwargs['filter_by_keys'] = tuple(filter_by_keys_items) - self.ds = cfgrib.open_file(filename, **backend_kwargs) def open_store_variable(self, name, var): From efde852a9fb94688360acafa6c488685d977e574 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 7 Nov 2018 11:13:55 -0500 Subject: [PATCH 279/282] DOC: update whatsnew for xarray 0.11 release (#2548) --- doc/whats-new.rst | 95 +++++++++++++++++++++++++---------------------- setup.py | 1 + 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 1302f8b671d..8e52d5fd31c 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,23 +33,8 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ -- For non-standard calendars commonly used in climate science, xarray will now - always use :py:class:`cftime.datetime` objects, rather than by default try to - coerce them to ``np.datetime64[ns]`` objects. A - :py:class:`~xarray.CFTimeIndex` will be used for indexing along time - coordinates in these cases. A new method, - :py:meth:`~xarray.CFTimeIndex.to_datetimeindex`, has been added - to aid in converting from a :py:class:`~xarray.CFTimeIndex` to a - :py:class:`pandas.DatetimeIndex` for the remaining use-cases where - using a :py:class:`~xarray.CFTimeIndex` is still a limitation (e.g. for - resample or plotting). Setting the ``enable_cftimeindex`` option is now a - no-op and emits a ``FutureWarning``. -- ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. - Call :py:meth:`Dataset.transpose` directly instead. -- Iterating over a ``Dataset`` now includes only data variables, not coordinates. - Similarily, calling ``len`` and ``bool`` on a ``Dataset`` now - includes only data variables -- Finished deprecation cycles: +- Finished deprecations (changed behavior with this release): + - ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. Call :py:meth:`Dataset.transpose` directly instead. - Iterating over a ``Dataset`` now includes only data variables, not coordinates. @@ -62,34 +47,49 @@ Breaking changes raise an error in most cases. You need to use the new resample syntax instead, e.g., ``data.resample(time='1D').mean()`` or ``data.resample({'time': '1D'}).mean()``. -- Xarray's storage backends now automatically open and close files when - necessary, rather than requiring opening a file with ``autoclose=True``. A - global least-recently-used cache is used to store open files; the default - limit of 128 open files should suffice in most cases, but can be adjusted if - necessary with - ``xarray.set_options(file_cache_maxsize=...)``. The ``autoclose`` argument - to ``open_dataset`` and related functions has been deprecated and is now a - no-op. - - This change, along with an internal refactor of xarray's storage backends, - should significantly improve performance when reading and writing - netCDF files with Dask, especially when working with many files or using - Dask Distributed. By `Stephan Hoyer `_ -- Tutorial data is now loaded lazily. Previous behavior of - :py:meth:`xarray.tutorial.load_dataset` would call `Dataset.load()` prior - to returning. This was changed in order to facilitate using this data with - dask. - By `Joe Hamman `_. -Documentation -~~~~~~~~~~~~~ -- Reduction of :py:meth:`DataArray.groupby` and :py:meth:`DataArray.resample` - without dimension argument will change in the next release. - Now we warn a FutureWarning. - By `Keisuke Fujii `_. -- The ``inplace`` kwarg of a number of `DataArray` and `Dataset` methods is being - deprecated and will be removed in the next release. - By `Deepak Cherian `_. + +- New deprecations (behavior will be changed in xarray 0.12): + + - Reduction of :py:meth:`DataArray.groupby` and :py:meth:`DataArray.resample` + without dimension argument will change in the next release. + Now we warn a FutureWarning. + By `Keisuke Fujii `_. + - The ``inplace`` kwarg of a number of `DataArray` and `Dataset` methods is being + deprecated and will be removed in the next release. + By `Deepak Cherian `_. + + +- Refactored storage backends: + + - Xarray's storage backends now automatically open and close files when + necessary, rather than requiring opening a file with ``autoclose=True``. A + global least-recently-used cache is used to store open files; the default + limit of 128 open files should suffice in most cases, but can be adjusted if + necessary with + ``xarray.set_options(file_cache_maxsize=...)``. The ``autoclose`` argument + to ``open_dataset`` and related functions has been deprecated and is now a + no-op. + + This change, along with an internal refactor of xarray's storage backends, + should significantly improve performance when reading and writing + netCDF files with Dask, especially when working with many files or using + Dask Distributed. By `Stephan Hoyer `_ + + +- Support for non-standard calendars used in climate science: + + - Xarray will now always use :py:class:`cftime.datetime` objects, rather + than by default trying to coerce them into ``np.datetime64[ns]`` objects. + A :py:class:`~xarray.CFTimeIndex` will be used for indexing along time + coordinates in these cases. + - A new method :py:meth:`~xarray.CFTimeIndex.to_datetimeindex` has been added + to aid in converting from a :py:class:`~xarray.CFTimeIndex` to a + :py:class:`pandas.DatetimeIndex` for the remaining use-cases where + using a :py:class:`~xarray.CFTimeIndex` is still a limitation (e.g. for + resample or plotting). + - Setting the ``enable_cftimeindex`` option is now a no-op and emits a + ``FutureWarning``. Enhancements ~~~~~~~~~~~~ @@ -126,6 +126,13 @@ Enhancements consistent with other xarray functions that accept either dictionaries or keyword arguments. By `Stephan Hoyer `_. +- The preferred way to access tutorial data is now to load it lazily with + :py:meth:`xarray.tutorial.open_dataset`. + :py:meth:`xarray.tutorial.load_dataset` calls `Dataset.load()` prior + to returning (and is now deprecated). This was changed in order to facilitate + using tutorial datasets with dask. + By `Joe Hamman `_. + Bug fixes ~~~~~~~~~ diff --git a/setup.py b/setup.py index a7519bac6da..3b56d9265af 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Scientific/Engineering', ] From bcb10b14d5c83375dca7a214f00a67c49f80049c Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 7 Nov 2018 11:16:28 -0500 Subject: [PATCH 280/282] Release xarray v0.11 --- doc/whats-new.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8e52d5fd31c..d7328ecff41 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,8 +27,8 @@ What's New .. _whats-new.0.11.0: -v0.11.0 (unreleased) --------------------- +v0.11.0 (7 November 2018) +------------------------- Breaking changes ~~~~~~~~~~~~~~~~ From 575e97aef405c9b473508f5bc0e66332df4930f3 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 7 Nov 2018 11:20:21 -0500 Subject: [PATCH 281/282] revert to dev version for 0.11.1 --- doc/whats-new.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d7328ecff41..1da1da700e7 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,20 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ +.. _whats-new.0.11.1: + +v0.11.1 (unreleased) +-------------------- + +Breaking changes +~~~~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + .. _whats-new.0.11.0: v0.11.0 (7 November 2018) From f547ed0b379ef70a3bda5e77f66de95ec2332ddf Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 13 Nov 2018 17:51:13 -0800 Subject: [PATCH 282/282] Add libnetcdf, libhdf5, pydap and cfgrib to xarray.show_versions() (#2555) * Add libnetcdf, libhdf5, pydap and cfgrib to xarray.show_versions() * More fixup * Show full Python version string --- xarray/util/print_versions.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/xarray/util/print_versions.py b/xarray/util/print_versions.py index 18ce40a6fae..5459e67e603 100755 --- a/xarray/util/print_versions.py +++ b/xarray/util/print_versions.py @@ -44,7 +44,7 @@ def get_sys_info(): (sysname, nodename, release, version, machine, processor) = platform.uname() blob.extend([ - ("python", "%d.%d.%d.%s.%s" % sys.version_info[:]), + ("python", sys.version), ("python-bits", struct.calcsize("P") * 8), ("OS", "%s" % (sysname)), ("OS-release", "%s" % (release)), @@ -63,9 +63,27 @@ def get_sys_info(): return blob +def netcdf_and_hdf5_versions(): + libhdf5_version = None + libnetcdf_version = None + try: + import netCDF4 + libhdf5_version = netCDF4.__hdf5libversion__ + libnetcdf_version = netCDF4.__netcdf4libversion__ + except ImportError: + try: + import h5py + libhdf5_version = h5py.__hdf5libversion__ + except ImportError: + pass + return [('libhdf5', libhdf5_version), ('libnetcdf', libnetcdf_version)] + + def show_versions(as_json=False): sys_info = get_sys_info() + sys_info.extend(netcdf_and_hdf5_versions()) + deps = [ # (MODULE_NAME, f(mod) -> mod version) ("xarray", lambda mod: mod.__version__), @@ -74,7 +92,7 @@ def show_versions(as_json=False): ("scipy", lambda mod: mod.__version__), # xarray optionals ("netCDF4", lambda mod: mod.__version__), - # ("pydap", lambda mod: mod.version.version), + ("pydap", lambda mod: mod.__version__), ("h5netcdf", lambda mod: mod.__version__), ("h5py", lambda mod: mod.__version__), ("Nio", lambda mod: mod.__version__), @@ -82,6 +100,7 @@ def show_versions(as_json=False): ("cftime", lambda mod: mod.__version__), ("PseudonetCDF", lambda mod: mod.__version__), ("rasterio", lambda mod: mod.__version__), + ("cfgrib", lambda mod: mod.__version__), ("iris", lambda mod: mod.__version__), ("bottleneck", lambda mod: mod.__version__), ("cyordereddict", lambda mod: mod.__version__), @@ -107,10 +126,14 @@ def show_versions(as_json=False): mod = sys.modules[modname] else: mod = importlib.import_module(modname) - ver = ver_f(mod) - deps_blob.append((modname, ver)) except Exception: deps_blob.append((modname, None)) + else: + try: + ver = ver_f(mod) + deps_blob.append((modname, ver)) + except Exception: + deps_blob.append((modname, 'installed')) if (as_json): try:

Im@M%*RFLyFhLSrmvks$!VH{@#2UVOlDlgjg}yiYwArv`Q+Aaj;TIc~ zlAJzZhk|Njy5ZzfKq=rfc7n+YS3a{SXh|YUcyG6xPXt03t-(lzn4GpMoh?#WyQjZM z%MM;kcB|GVr49xfg#Q#sfnsB$bL`1aa}Pz#aS68o>orEr*r!!%9>k?_Ph`RC^Z?D! zwyXN>hBE;!gob*+$;8>?$JPVaRq=1Ps;Otl3$nL#Ir46Uy=`|c3;FypX)Y>n7%JDv zfK~gTT*AXgnh%+0SpXS|CqjR%Zg_zjhadn4LzJUHUzE+<6QTXZu|japE%e6W#`M!A zl~!yO`Lu3WMcsVCDN|T#g-E|yzn2^CbSrbYIdE5$YIEp)ExSlb?U^~G$O80$0b4`- zTJV0w9Q#1IjXFw`v<@F&mwwro@Ry74`7gtTx-|^FZPqVlKK*i;Tz;0`4jsM0w;rTi zsxyy9JGU$`Nox`nMuZj8a>(S_5pD!{}ECi?$b6q>PYb$kCQGvSzE2LICa})gX znzR)GT1y6(NJ!`9(&`Kn8xRIq^$8)gq7BB|F2PKf7L5Dq@1XK<9Jb-$d1Fak~&>+xP6V*&~WB0VJ*!SOd^{N#2#CdQ_Dyh|umMf37NEpxYN#i9%D zrKeu7_b<9uXtXvWe$%jLzG)e=N3z}m@msC169U8D?YPE^Uf2&wPY4G~!&FTYx{1r| z%m2J6RZQwPg$Q5#boF7B>wu(WM|>SzC$sfyfhjzBgu?3k^#?t@=&p{7J$dS%P`PhN z##SOEpL}>UW=X5!(9Y{DfW(k6n|_r)*zS_GgqSp`g-rUZS5ImLw4WijB+4Ds(=G3alaD;evxo^cW$r6mbzblah!p;#8($d`htnR zpRSKndOaRD_R;Q~XJx4GHh2&1e(G5ciZ%{ZOkN4v1Tnr+q;XmX%P=5i*0SJ3`t!zB z?LYw7Lcc;{-SO{I=-Sd5B*iTzK{tVhwXHJTl_v5}dKbuUuA!F<^t{KeS z;3y0=BE;CAr@#%!@Pf`#Cnv_5SI5=lPvCnqZ{haliV)tWBI+YNuiql)=ogBalCcKj z^Ou!N+_&S9GiJ9B{)Fs(#pf*sYg)g<^1 z9Kg4Pa|sT4Hu^%`dCObe1TS?{W4u1pwh)txEWY5LHJbrR+w#8M=V3AG^k$nXZQMM(+rAvHrEPI3bHhaH zdy=Hmn@JEjaGsBm8gZ*Faww%AD%OLSRz3b4M@S=!3Ju;}V3LYUxy{OLApt&XV3iIn#1o>5&JI_@h>SVRSI8)R>F zU;ZQRSty0PhJ`m-)Tm@3(`zD-9BmwPg}is;*G7@pgNq~7i4R@x<%~*wt!)IF52+h) z4nLV&6o9?i8I4S+LJWV+^A;4yKQ0RmP0Ob?Xf&3E8!QN{)O999tYy6KPcF4u1d< zgfL0B#x~JNS340^?2Cm zh5zU+aNR-yA)NGnkl(NLIhCtcLv_~ch103%gH~B^@<#Z}Yt%zog~)L$VXfI3x&S&> zx3wZVU^&cDh0qH`w=_hyKRkTu9ltyq?^9;fB0RtN`RLd?g4x-TrNF8>hVQM!#v@x^ zrC31!2j*`b`Z(tCDWQ2DrTxCYj~EN{z1mpg%B2;aaw(`eRIY@oAdi+(5ju9!BkIEt zLfOTGrC^{X8wv3)IY127>DBqUOeKf6ZJY>RRAoq4u=61M1KiA^@3&=fEYo!?;Qf0O zg8#?YSBK@@>we?Ib-24b!(E4LxI2tt!`H)N*8sa@RH$BP=$(|OS~zIPp~tyvPGqFX#ZQ{8w)oqU&_$;ju|`fqG*6CY?ryGl zPtYn;nQ?ODU3Wn=@`l(<9-Uswry*m`=fqt(2sQXdA09ulU0{_HuuyY0IuWz~zz<5A zB`uVv%tvg2Fuh_4*Rc0%0?vicvUQ6v_BKG<5S*_8kBG}Cp-lg=TL(|@VjHEVb{hu~~;|8D=q>KYPa z$m~+W=tE0aRiX`j@lD*)#YpJbPzbf?(Y8=Cy(ED0kAbX|oTB>5G^z!<0< z7UpBx%Ip)O33h8#3RZ$;6u|0v)R8_FvrC zkIFDeii9#8u~c`SYO#uxfrtTxX2Xy6#1xea=1N$hLKitr)@$_$;we-94f2@HTU%?u zJtZh7l{pz4pwa zR%>*ARo~D1;oIxjM!V}J^9_WV_?tT&`Ph9sn`F%CQJ8%#i}4bpYB1>eqUc#wq?sQ2 z`FnNq%urFHlj25;0y`&3EbC`pUa4eCnDTOs>?CxuNW^qYA@kDrFw&;RBef;Z(e#xUK$&>f3o~upPIm z^QfA8&Z$BC-mA+tS#km$_N%@9sUh=wV|nY#`t)F-Z+xRVL{|7O(PTgBG9;5_&lbYA z1s>^XB7aI~phZHAbuqf38s$l;*OY36eNZVgizwDsCt`U*lBkw)ObEVg4W>fP-e|Xa zk37zE8P?QuNJdnwJ|pITNI_710jO57EfvG}m=&IjlQ$1y;)vA`hzYw{sW*ptD+|NL z)V@Mn+zpe}%16NrQK8;M^pND17K=z;>U@YNxWi~IR7hD>sS#^qbMPyhrq$EeY}SG6 zF(av~KJzL?xp6Khk9&i9-&zyqBoK}eFR+|>hYY*1?{pGaIi$<0SIbnBCb2(?F5-^< zEXpf!{g^dBfgM>>MGKeXH7yo41mx!Dq+Y$fO>!5JCxZE5UfE$gSA1(c0uAeCz1tUw z!-0269lGS{Y6uf$PNHbdF!knJ+48xYi%}h+NpHq8n3`X@)$gZCJ0`o|3)`{^d_S}F z!fI32Z}@;7W7t7$`;qQthJpIL>IeJW!cvZjOo8B-~+#mTPGl|mr0UHCfQ{FD?O zdD(+zXE}_CwRW_-yJMr7ahq~7?Jgv{aSX4B!C+{Cd-b{+LGbh?fhRGAsCG}o-ZXe9eU+2?K_i6JTPK{8{$orlQ<==AspgZF)OT3ktpj9m4+Y-+Jqt%{<*LG z9OKd>VFt~Puo=y!@Kh_g13#>=Fl{*F1s!;4&T{{q2Jy|Wne6TAwpCe?#k6-3VInxE z6`iS7t{pjxcDRxiG}q=gEUm%0yY~C`PIQk3?v3`mi7A;F737}x@F9e!$pTHFYi3H= z+x5#}o&A|=&*vGHUm7XNTF%K>TdMmBvTC6D(nR1xObtF>Urk12sm!WpF`yXUeSXud z@ZejHYT8hVpj7@&>KjniSZ zqxT45C(dS#P?KC*Ev7;fiWT3xD=il=xt8HUjY`U1 z4b-tOAJi&6_(j{{$cj}wyp&MA(yo%pvJNi38*~5kKThv%ZJDlC2=I!nkXJ>S4NP8p zN4(p*{1WS^WN=gyyRg{&PGtML^{TXORw%S{TClfH8Uy>m1d8(Di>*NEQ~MJ1aKle^ z7Yd2ZSK#7c3g@q zqy(`j!F*Ar&E~%cU3o#h6qT#g3OfOz3YCsZC0cPQm@ke9m5*Qu^huh1{}f~9e303( zs`3Q5IS~pR^e*I5l0X!2vA2nVan}5467A>i-aJkjFKGvp1{heRmoLG}AOtzAqZ^po zOD!7CC`w4(+tO*D1!bqWI34SRx5YqV_BzY?SNYkJkmbn zs{MQR%BHQa5)=S-@JdWtkFIiD;RWgu(jtw^o6Swtv>ct04x!)Lcq1y=(vI*M7bL`& z(e~x-%p~VxqR2dd@wMR)yB{GR*^pint<1L4( z>Z};xu+AQ8(Q7G?f%%4CuTXDQe`i@_m+JmW+?3l;ZY_||Af_^7yk33i-*O=C0IYjf z>x>Gj218*nlC~Jfi&^G6MqT?*9)i7CpJITcwo3*qT^2)vWK9;~y=W;zYF!vdj`weR zAY|}Ade1kU%2lo(CYPq8+6FfqD(Yl%%C`ykbcd}lP90SRmg!&uWsTn8a9=M~zwvEy z02kUl=2%3{d=;?_(=59`ls4@O5u_AF8gKe36Y2$-4gv27X2n3v-lf5m&N@hXJO%HG zX!gIy@0+_@>uckw=lZ!c-@Al$GQ6kPpWSKVC1$v7eAc&nr-M|jT2{3;ba+G|F0^vg z!>L!FV(TH%{~oU|a)Lu|$MxtnNqN*jOST3AimyD%#uVPNbMG@*lUX?HAUVo)C<148 zqPu$9s_v26g6ddx3H(z|FI>wZ@XBB|eDRX*rvdkAQn8BsBR>0NYuWCU^bR#v0X`xE ztdKCwqTB}4rt!t(z>um(NWuqNb^jDGpL5oM*ev}`!!rD=rnECDahXijNZOWphNJL! zoebU%Tl*@Xd}37B!m^il)^n~0eKcN#mIRN{+fF8aCuN2DudWTb+PbC;Rs)9m1^$^D zMW=iy9)-bFiVFK2l&h=@biM<&)7=Eq?WBRPjc7P?(t<%{sn}vScyNb;&3mR6E}5O{ zH@pt)W}SASu7`TY4?hZ@bMXxfJbP7f$tEUT?HIPxw|<$rm-P$XHY!4EiaCmq!!ny!om3SS;F>KTM9+*}A7It^OoaV`h?J$R( zI*gZovx%HWRy7D0JKx8xO17rh_p+wQf05{RIjnDzX}DJOSU1$_U-hpeR{q`;M8#)s z`t$RNx=Xu-%Mx~B)%p|IYDXpIii%!qP)^MTZ%6v!N?f`f8p+cFHQG-RZWKz0+XaC zd?r09=&9*Z_u*ZX`{8qj5&>uq=M#}-N-C)e;#=ZOB?HB>Beur(hvbJ+QQnAEx+S{l=j!O7VR%Pr_P9_)65MfOS2>h$F>N~ zKyr~*h*ciotf`F3Gvjg|VM6rBG@U^N9ZP4iU(SZd^vNXw;!{=icjQIAi+6b6?ToR~ zne_FvT%eD+EBLyI*If62a>&Jms!Z?MuK6miivG=_8`u=8fY71)5$!K>qK`3onkspN zKV$dAcCzlvoCAV8MiPMi1Y@ zXJ3v9&0R@>E!Hz1*7X^9%}V)%i!Q{Se%f_UDu*#Ya)-#Nhvbi!y8x53j_gV8X4H;} zfxrURi&_nJ{2s@vU?mtr1KhQ8>CV+=WZ8BzT$A#x$a(hRWFD93lb#JunB(Yb&MP+3 zFU_L;x>a6KvgNh#b(NvzO{fY7C1Rv6tM3lgAT~n64c5xP4;4Mrey=DM?%<9%d{qcC zN_aAyqj9!RvOL~O#!21gRK-up!5`LHju4Nr_xs$x1W(4qf4{)i`TyJr;J%+rX> zCH2wzW?p^#DI*iD7k_l^36^<4syH7^R-&F#AQ5Ji0B4=`)RnoT6A(}^wa9A5h+D=PVtqaES7d}-P}johiiZFzT_*hpN&tXhRiE${b3H*B_=4`$3@bTi=7KSi@N1=3>(a)-)vUXn!whj_w?(Z2YEy|?xknEHNd z7htt`*XXQR&Nx*Dzj&8Jq4ML{n~V`xUhZI@5WvOpC@g1&@Bv;DDApGvAyImFAsErX z1}1zvj(`8{R>?s9w^TE1emM;lsm^22$dfj59=j2|lfEonLSBRGWWw z%1?b1)}&i+Q|LhkDF99(3wD2xFq7;9R(Kl$vXpG2d?Q)o|T$9$g_ui^$OM}*-)BL9_s zbhΝI(;K*U7)LYlQo*Qch%?Y#nl;-5Nq%W23Axj=z2XSVZWSpq&_aa-KhEgmlC5 zfi9v&B-TaZwYa)QIkuxuQ`K;CYK#~mg`S%-%($0GYlX=^=<3d@dkVl+A+5e#rqNVY zQ97VxB0o}BMft#-2E%|)bh`3!8~))h~Y{ozX)`l{hH zpO5=n@7Re9yEF#iwHB+fE71d2;sSaAfP@7AAi-uO+X|2AHKv-OY^p{}6X9EidCUR~ z_~~g-exlC@`MHwlssME>7}YL)tvdp0;5a2gX6@Yh+&;p}(rQhE79S&p zf(TZa?jw07h3&BKM1}?uopieGQuo4acpI6a!U;mmqbhC(kCV{1OKN@F{T$JXnH0@e z)#z8Ddf<>Wf{RA=^w)@W+Pc@WFV5z(I{n3Hznx(oDaquid~7KqbR7+KpP<)OIbf8< zJcHiDGVF&lxb`DOd6OMx&W2BoIk)A=#+~3ZIBg5lJcGNklXw2?X*X-3xBIPn+~1AQ zXRKsJXL5*j&n_2uD3!@e&U>nQ?scq{R|ylOCqDME1{S4CI3R2Y&&?N0J}na)Uly@R;Z0j04f8ye*{)o5R^Z2n+%zo zu2(9<-N>zXQTzm{s$j8kG%Pm8@Dc-awb2P`}b+4{bhPYgO9%Z`0NvJTq?nf@tV{C zRs?buHs6;oatT)y;%}9@3%_P>FF}Fvrru8B`??#ew=h3H2oAnT6n)EVl8^2Mpq&k{ zeubm(IV2(~&hbY>DGhVZ{;rM#x-p6_UGru*ja9(FaI-5iUs5-Df0!laZZrc4QMU8D zjIo#N_yQFuA$B5uVWP06L27{xdvVx`akI4(M$>C9qNIMg0k-|sh*Y+7^KcMj$Q&Go zRY@S4t!4X2nn9juD>j|~mA;W5$HZ&fO;_7HZ>orOs%SA5qFAW?nzA_C`~!Dy7x?TU|jDK>9( zuWxwJF8o^#7lA?`9nFfYRed=g55JYwQB2D?ZH+ zfxtXUSQJ%VVaW^(n@yIMq94ns#G3LRkIa}!b0Iw`xzfn0sZYW+0y&ixv?u7WPtd!o z`@zdszt(c9Ka2Dar3;;=G(%HXoacwjM^r?L?p93_Z5|0nu5sq<#6=?uz42KPxIzJU z;&s*M>S&hdDytB2CZc<{)`rb_U?$K~nq<_M!p9v@ZRw?7Z6gx|ws%~na$7>Jsw^~K z!T4$_RdTP}q*Bm64IrBfFHfM~@+M5o_*fxr#*?K6@$Ow3;%pt7bC1}uP3aHa4ZLP> zx(h=n)JSg!%jZa@HJacPwlp97QfPT1Z@}2yJa2jmKg(LYJmx3Ol@6m_N;q`;QD?Q- z(~I`bM#7RJ43}9f8l;rSopP6yA4gpBl8^$P+Z^36TM0k*> zu^+=RAZx2h8v`}JafY>@T|RedkE7=}8kTSr`|jh*Z4Jf!l%~nD6%zV1W3_7}8=~VD zGbaj4te`-rY6&(gCMZYLF6w_sMo`QwPDAt_cr_V`++AL%^4SQ@pe_GKnbTuCy7@>` zWq;jQRT?V~u>jf-fIb(to_Ef#hv-a5NA#fhrK-&VimQ8!sdiHQ`FN=s%VV%$$w=zD zxjHOxX>^e&6K{)7HWFC}VS{F!3zf)cM7BF-Fo(I2E`r!-hWy15%I2l!@fyupD)`F-~U9}^K zn!fCOHD4|>#+jrOJc5sP(aEyih(8keV#fw$yY!F#C|h@?%^h{Lj3{;?R1)R*Z$MD3 z-#F1)U(h^>gEUtlAO;jdpxZnbLSi_0t%)a9J}$+;Lccj5L=8Rfzhv|GvswDmKCT5x zYu5LY7uljPp3bR1b@t{=@b^SzxswTLXkQ%k@4S6CwX;@VD)T$MJqn41;!{#X#B%??9LwVbsbzyhEgt&EW3pTsF!pfYM>`73$jkN|HOk#PVWmn?x#zx9KvNiNJ58P*OuwKY4->@QZ06f2Vybd$OTaJ#T zJIK&82n;=1c?sur6RLjyNw+w_F1X5x`jDe3qIvDj9J6zRJKWF91zXHzd^_ai)tm{{ zN_x@^wnG|}m@Q4pP8{{3QadKH<=}`n!2fn7Xd~vr&qhSpA=E<{C=|stR-(K@2C;PGT*f|28FQ zK~$Q(XDS#6L$Vyp4VdjbVvTr?9+Mq>dN=XEgY51l0HBjsp--EZz_ozc)VGfi{$8E2 zksgGVAD~qBLfK7b-*o!fzxh8J!~NV&%?iLjZ*lmFT>Wn>fVzO8#X4N|w}nW;*GC39 zGW9bt&imPL zmmf~i)nxg>PV;u{eXxY}x3&5E*QYZZy&Gw&AFrS%qW&L`?Tka_75{Gw2*SFL4(;GbXwT@g6doE zuws$i^1UrvsWLc1S`(G-#6aFgkQEY<`Q7LG4t5v>O)ZpK{)G02Z<8(2doI3zL{9zq zKf3u~b}v^4p#67nIr6@#z+Hx_pHz(>G92RM%~O@08l zuE@YR%chX27$J0F9-v1iMN1fzH+?SoqcJW2nSc79Khb}~;X;Ccyi@Hb{bw6<%%N!ihD7<=8r9b#+a-?Czw~q zS$j3WJ@Sd}UwD5Q+CPRF51{D!+jegi6#T|tw{KTV{j&(YAsEPh(~f-=n`4Lc5Bt{3 zrS<>8v8DM9r<;)3Svum0SXXEof0?{#JF<$yp^ z34+4Pd4s*r{?DIOi6G$61DjjJInYc1ggr)7!GHeuU(J61Dc1@HPZVG2l`@h6>d!K} zTu%4D2>xz~yalxEcLx8iaDgotTrg*#PW^v-{s(oRT83^AVf8LBsp?mNI1zhY?w|XY z0fj;ig&CUm!_)&IcBUne{6C}lH^__Hbj7=@YoE>m3%<`)l8`qVos{>hT|{1&l!V2TejL)S%$dMujEn@_%Ye zgb0Bp80;q=oR;)(8r-HbK-Agd7gaeXIJEo%6&>oI<%$dR`%SNz=NT}3|FwAP zKWg#&;P~Hyk2Pwg0KOtawk-5#sha(k>SAF?moFeyFN+bJKTDMskm_@z#msCtAXRfz zjNm^Cz`FwIzme^kX!$p*n(+sJB>Fx${Wq(1Mk7!FtMEkb(tl<(^f#-U-$R;m0amT( zM;`yEu@`gztGrE?@v{-2{#5437(st#l@RFndHp=^{P&#c$QFwKnSoefG@=SgrbmG7 z{j@np_~TUh6k`D1vowve6Lt{L#0E-pkv~!GZvw`DvC=88%mam-TgoL&^Mh-TY(3o{ z0z%LJcjSld;L+@P*8!9 z3Rla_0V4PPMV5MJGL2!Hhm?C!5Fxy7FFv`zLWlN0|oeL=}a))INnmWvtmHwcUJTHWdJ{%2PpkOta- znW$#?@M-?GIn&%rzg#F}SU9*Eqfrd^ z`;DOc?KoNM{h!3!ySu_mUaKy9IZ_EU&Ci#k@?|RYY-Tgi7E6r-y04c>(Fg|S5We6p zgR-D|2@(bcu*CBTl>QRkHoSz0k~B3^1?OZ#O1>CBPv2~JMj$lcCE_)nr^!#8It%epU$#lBR5;q1Tk(D2WI3zN zF7wpv1-Yk1_?y(k7z*O9YRr@{!!^>5M!H^2zIoN9&16(n!=yhD(C*UrIYkO9ng2x4 z#?dj3n$xK8ZdbOnCJ^>}8#~Uvd&};H?8j!*2)W^(?=S7h`=mL!EA$-s{!*hE&;l zNaOzcuqV{lF9p(5b4NeZWn8LI<*3$jq^^aoCEI12X*2C3wvxJz4Fkrh+mNsLoT0}v zu(SNMy(}AMTlti(_9%bO<+@@|cuQsEdHQnF+#z0rD|*W} z-kMkW_5&8lU~vKd3;NIn{uEQG2mae&(AKms9~SF3_DD&M1gnE6FMFpX`C9?HC)wt! zj|cqld6q5gcw8(IQy5eWVS9l_dRdit=*))+doc>~O{|@(#q+MTEpu+~nhCecZp;7b zg5;on`vJcQ9Svo`57cjmC0Z4O`UfK58r+?(h>gmOmns$^b^=3!xr{}wyHW?Min?Z>z0kS-(&?nn+ip`)O~vz5?xM1&8#8hYTMf=xa*3IXB+^R zMdDfeqFWYe!#*<=eVlAyrmgNPrXy1-7jLI0fl(f=7(1iYV;4k6g z-@{%|UJkSd$lHqGAHA zMAYTb!@ynOrcw)OvPRt-U6wzvK)|GcI!Gw4C|6R!KAvu~;M}f4g4gQn-y;0%_kd@h zg1oLxB?sc_*~=U#9VLOCO0SeSGk>LbHa87EU56 z%Qi7jI02`aR*|*AL|G{ zCE(?bgq{AFxYPQorYB1suUh}LB(XUoRwP;6HUVyVZa5M&L1A>zz)TqK7z;Y#l2 zlDpXDX6I7IC79961>@P*kg`E6o;FuQxu8AvttQJe_9f)>C4ke>groD}0#lt-%$E$E zq4xtdg4~I)hbgS=YcL+ydKx+7@a7MVVcOh@!r&!zDs=pGNc6dt zNKXFrW#;H8J9msK3QfQrsX{z~v1QriM~#Tv`AU>PZuU76nf5P;64lm2>|>kQ_^Tbd z!TBJYDPmo$OB&S;7A?}9;#=bmpDU3ES)tQA&JD8WkLeKW^Jg_@-9pdjM_xPIVb+Fm z6vgE;IqTaf?k_j0LodqGZ@gM+JGNSib%-=iDJY|HNqa>qMZLCNM64d{j-w|KFhsmu zovbuw5nsO<-bn&a=q}#-mv8ulp1QiQsCr^>Mm;c&D7V88k9^qt5c8-iJ;e~Jdp_3L z!Rff{kfB;`i)~gkBW7f-V1)=Na4a^rK{{2a!87#XF^Rjmu(iHS>&Wm+E5iw{JcoAD zos8w*B7$H>AOzSfE-EYM$ZSa^^nwY@KV(jbz{IqW?QZqf(L- zudcEmU1!*uF}(Mm+^^RW9^+sm*T>|MHUmCQU07bVdfXo+TYHHZMfib$`ul=oIVj4O zD@;ifJhyJI$lXG4&bvg)bX?y;T&EqwMN>&~af}(o#v_D@&!5;*)+RB`|I(1X`69Ej z>w!|S+Fk&o^YgK^VV=CN#^a@9$^svk@q@QqrJ30K*K7xE^Qy2fhnW|l}z0RuZWM@onust+_eCs$2NrtGAyV zxxj0h<5U6Bz)Owqwh_aV+Uy4gsrI;UsAi+ZC%l4agR;iDW;jqbUClOXSX!uzm;r?# zk_7d5PzhU?l;iaXwdX^BdInFY8#pHAGBVuTTIV(|Rz1jtMiX!Qqnh?z^i(Np;H~9W zD6MOln)i>MDE*4L5ZwX>Czf`gcvnC(gixS={+HVL(UCVVlCU;=LFNDkq0~c;>;y9=Kafx zh;h1$$B5A-1qqG#kI=38I+dsPR&Txttc7~7mT+aI4j0=Z2-Vlazs3jhum!8vw;28s zR1%&3j^5fRB35A&Za{o-c+cw<*xsQCBM?2GSb6oeIvnvT!#x;B;C!&YKi|lnfKvtO zxoD6PnHk_fi0K03Z8GDre3OPe&T{L5K;|9o)yM+8{1ln@I(C}8)BTc~vlsoEBnJLV zC(?~(!&6rzd0#z<_w6Mb1ZPHsvE@;)SSF$!SF7Wj>h|*2an_g2Bi>#gEg={(@= zeleAG8$B9fPks?KenGqQ{J(klx5`4wRrSsz<=})w{D&}V_rpi*)B;l`>0;F zo_04l@?I1pqkdJ?1D6^AE1aR9A3-!a#9JF2Nz4J)eCV%NZ^mRSFj9k}pD|QyEs|Vy zXm6XjJwH`qry1>?eTxn#^W#H=REX5~8TFw-xY&gANGS33ZMcWDLijxRP9vy&PdNu~ zgbUUI|0qH8rdPm~;{s3rMRD=h)pf@W=!on4jp8H-P?pSFh<7u139Uzw^0#F{OW*FC z4QLj%OAy7PW6)&_4Ns8RDX|<`z4bsgH2*q_48ONXqhBM&r4n{Pe3j1y%?mJ88-+Et z_R~kGW+Oj-`_5x)-1cMn_)Ce`1ny8G5bUVBCC-M2OiQ8rUsRWH!51vC4=o z0DmJr=$=?=!?3a(gF@``o1cH={uQU`*ynK*7oDUF1Y)U~t;*X<*gorw-qnVe75jIM zj0p5~ZQ=r9OGjiI=Zo{_Kb9T^11N*_2_c3C((YD5G5K#WaEG-s2fC(~TzDQ%CLA*2 zn#|c#dCyZBL4UcP|5CLcRaR)~!<{_4_|-a*sYYflFk|z{_zSk+&&bMI_`8{^z+>>i zpZdLUoJ8RikHCTU-Z#j zdt7F%YABok6wmzDwZbL#S?E;91=f>&&hrTIvxzwz75?jLTsiymwke_DdAa`j=V~+j zaMxl?g;I*#^LbJ=9u5&~Y-=uxA9w+@^#{(daB@_Np+;l4L8ePuq_~0ne268OBNC`m zC2H~~$iVyd=4-d6eawAm!7=ifegr2`6kajZW}n}DPtL}^hu^HV$ub1xC>G}WYIssB{sW6Lry&B3!e zu+7!hj{h>teLI-XbJ&W+?v$9+R$E!Ww`MJtV9Rn7DTAF5AB^_&b!#K(wt2k7mY%?6 zn?FCarqJQr8VfI_?0%d0d!>F-MuDgq0k;u@mpZva=#*ZQ1OGiO{1cW({i*)W;^Xw* zp*_MxSLGOxUN}%N=vKoY6tk(TfjLfZV=fU{L#vS-I*Bn1nJAW@UgL49u zc?P^Eh-~|-gh=(JThF|EN6H2bLL862pb$sq{%(KC5BS{w-kNtVvX+`FssQ037Vl4n zd^vDvr&SlqRj;=f5^X;i7#NeE*`SW6%h=Zs8`wTXp#C|if(zTjx5Vy`E?(LaPo)=8 z1~J!Xlq7_bp8~fIGC79GVzRThZ8r zJcIw#5MATQph0Dh!P%)po5xOYN4{{W?#?}io?Q(~#LMYG9oQhx-Rxlu)rJ0o}#8I>fmkWi@j67FwjW@reepEX@6pQV_h0SKHn*62LtsNA~@ZH{NOH>QbO2fmF zo4$h67TO-9+SBIN5bruHiI$Bj4n_SPpNEzCI^_iLaO)Jz%O>1QLeWt}`u6aH#Llpb zv2hhwvVKyf1@q^mYgQ1jjX;V8|$$W9BCVQd0>TKVd8+|i~Xuair&HdT#5 z?4N#S7Y%flrZ$`N6mR9@y%gXvoQMMZ;#^kb+(MQ63YnDaaPtwebz9yPX4rYo(b6-z zDmpmxw>|^?6GOsJJ;{)jD3pL|5g_I7WAbt~S`e&G&C&M+>|Y9cY304C(TCPnY**ld zuJe;=-AqBgi$9)?p=vdZw;kR{NE;~*eL|SOuUfh|U&Cx&)NwkEN}hhhJ~OFl^!ozw z8N3fVU8}`X+^rI~(kHtSy6JD4gAQnp=Mwv&nFe+SHyGZYn>3!Yvo0^tc;Us+tzaGajGSoo6XnR(&72ra(ql=WSBd{R9+`o&wtXfnGFRs)9ri5b|=m>Sd#IiY9FsR zhCkPnI>w^W%W9E88JwRrKD@fl<5K!ceW!Q>xQs}7536bV@^5rb)@KQkuG*CCN5|E( zqa+rph>j1mPdlXz1oAlqTWv_Nb*jUQ7^jn5Dnck6;b*OeAjHk)0TO@B&DQ)*_Ln#5 z!6$T;Ln0e+IcICD^gtl5x}EVURq5b!tU8kd_-o4Wo+iNup}cmx@SZ>r{|uC!$0x=G z-rSW+m0%W&!EZin-C*CJH0fp>Q9bVoy!AZZ9NFw7sFnupH};_kf|M5yOtMHNQadd+ zng)Kqu@YS2ZiqfUYPwGBTPOiCJj8A%5X; zRoShkU5$O#m5Vu<9Viux5i|6zT1^Gz8PQp1pQz6mF(#9JCnj$SpW-JmSGPNG4}GWg8x)R?#y+i3^#@`S_PO6F-XBuxuT_$pV}-Mo z>+21Po&?`I5R3|;A}3RT`mCI8b>RPmWC7DpHV-9Tfrj`cAbv|Y9a*&~yqkFT6%A`h zWH-;SM#b%NO;)0mM{_>P01A0A!d*vPCE1Flp-x5BX?XT7#bd9fGJGV>qxQAe?(_z8 zq=3DLFZPal$Ihaz-M zyE3&fyYD@pBEz9=D$KU3Ui|E~<^Pry&ea7{pZivM&D8qputfc?=YnnfzA!{u9yhb& z0Z|iNdw5D^D)@$j;X#c@+tc6hv6oNByJ!PqF9|Ny(RJ)_c>@*s-#M}b1`m3n$QEDmiH=Olkk0J2l_V58}KB-MwO!|F3;r6^b!c) zcUg(HQa9$WdV#ER*+=Ww)5BF&;fp;qvL;<5r0739o*osS^DX8l&z6v|j&>xM*O)se zp6xT;P+6X@GtBS1Zz~v)8u;ux1MU2{##Iq@9|OMtPlVLB`;a)`5H5||s+z4?jCL#9 z-D!~1Khflh-HkWzB~q}1l9x@#oA>9&34FV${kWuz)ApS3(nv8kSp3~_K9AK(V7@is z{oP*;^L?-bun{%rfHW+At|1Wo=?nH`cWWR=`W8koC<07UIWS4vqlyQ1d*cI}17Sw6 zx%jIkp`}VCzB4ne#oBF&;UvXAVT_$vUXRmRATU_zoHnRyFMB*Nc`S(g z9L1t(U`cGbf>liCdu(kjb4veN8laSl+_sh#RQ?uM^s}PyzJ}%-%56rhm~mWH7eWTk zy)Ge%++bl5GwnZh{qsS(HfZ2o^Q)$XERZs)<|i6Ze%SDXr&tZpzs z$5S4Q7hpvyUp4M;BWZTfO*tlrmb9H>8F?{-L0bex4tA8NOlwG}nIgyf^pC z!&+1PYaKdR`S%Pr>=;xp*Wd}t`GL`!*ViJD6c3hsn%Nvt`DlJDaGp&-h_D1WN((sF@{#5u>N zAfN6Mq7{gcWCs!9uU`_?XIFIX9ELRPZ~5+l|Hy(HOSCj{6p!2X#oZn%-nh7}3CFn= zy0SL!-8eY+wxL!!HGSFk?yC#;FDH~2n!}0CaF(63a-M4Q1pk?9%(30^66y2;c2d0k zS{vBIw1oo|Xg7;^^E@$GcHptfb{MG>8o>ABWNrT7tRs}-jCeaUe5Ld95Y!&y?Z1bv zLfg=p37>mkw1{Hy^5vS5Wk;^ne1y0}Vn-th{9UIL18UjPx>6C7!tOPP>WkYC>}Nc7 zo^3RvLxL-t5dw0~oKZ!@3kQlLo)(WMX@2i&s~Q>odj1J@pU<)~5=pcPtq==JOMPmp zH#2&XildXaF(J(y%2i z?WWQ>!r_&HLX>5DDHLZDy2nqSv`TI6{+Zq^<)T6u` za0#IFUKhOeS|&xzXKi~XpG7do-!`1|pidA>^iaGpG2(Da)CBfZl-_#0x|-L{ZFH{X zN5X8pCAw}vV!+mmp5Dm=L}>g)X*^)yy(p5Donpc{Y)wFNzn@@>O9jSi*#V!8aDd$?o_KN!AJnfKh#HBhoF_P46j z?}}>h@Yk?0;Xy?ZA#>0V!N^uU;1!-?2t?`p9*bP48}}DmzxPfIE!(K{Tq{RiZ!c9c zc{M9sEE|Psmc0R(i$9ifc%ACn5_k2z<#iSXdg>@l)v^V>iZ&0=%*jQi5XCg;3w>v7R>WOq3!!Y~ z&8IT^`I!Xg)=v%(KanAd88bGmT75yD)(9xfX}{gP*YL!d7hy^_cP8<=+SZJ@ZDgES z$ta8ow?4B(=CK05sV2YvJMe}{uGj3_kl_} zoT_(Mgaw6g$kAyNOQs(!Jy9AdgH&t{$BX*hV4uVD9?w}K^W)mFRPNkKGRGzRUgIKx zxbA#U(`g}yd4WGmFS0?1FBDLOLQ~im!YU^uB-3a6u_Yo z@;ic{rdL4=nWmlsHL8TmJ-%|XbUAP}Ity0zP?Sm%H%@#S47aF6A=8H=8p{18orQOV zFr(B~fJxynFME{`_W_Bts70LGq=2>IZJ2LJ>zdL1?g-Lkss&R+%Yqdlw#@9A@$Y7g z445%RDm6bWHekk|)))k{AyLdI=X=64s#0;NRU`3KNxbk-IIorA$}+A*wwXzh^-MrRoZP3M zROQV2)y*nqEH#*u>vx&*L4EMvV1(Ag<06*Sxr{CkT0d~UcB;fnYeTQlK$@P#|LChi zQ9-(<#^B&DvJD0uQ)_E0apN@;Y2pP6aezMW&~BE%;$U4mjb-{ysTt1$>J>rtxdrd@ zOZ&U3iC>lB2E~4u#zhC<@QYk3j0L32$wgMBx#yqt?T<-elIhMNWEg~#-dXm2z|`@l zRjfoXW7hislA}!BuywLORg5^ov^LaTYZ& zO}mn@5!;PkVJQ+t@j*^!k=1-`NiAkavr2HShfoFtbb-%_v(Unhj~U5@rfU>@-O!28t_f%knGj4;8avqkBkZli>ROicVF>Od!QI^nu7Lo- zJ-BOd-?#?}?(Xgo+}+&?9^5zX@@;0$xik0thWq`;!?X5U)zxoTRad>$tru15b2?IR z;uj`6i#x|M5nfDxNK9v29*DZG$;ECm&trPkVF=i`7KmW?AAm_z%_^(f<&I^!dCql_?U5gm@uQYMKIY1SiHOMbSr6xjuS5Uu4wQkIi@Z zK-j)K-QLr9 zgqvKH{z@KP*MieY{H=k$VvgPQQSxCUX|Od)*#TI8x0?5`1rh6mu39ZQc8zr^|;y{A@ce5kEAdH-b=V<)6{p` z;V-2kPfLV1C#T^(rzh0UqE%(;KW9I+DxXQBH=@BK4=QoTXe#bsI&ZktG*5W}ojdu} z5Ubi@UguJ?UmTh&QLNhP~z-A6&E4 z3BqQ1x0PrTxLXjLGY;V~nLn48&Ucb=pv3R0(e1;+xh-blY)yD5S5yPv@?>FL%#1mn#P0q`}c3>qpE%z5WDG5S^a7kC_+6W|UiRJwA zy$uiQZeVm&_)a32GcF@gLOD8;YMb_0<&%0`o%p~V=GLEu!FNjKO30Mw2=ZL}3QD29 zl=y<^=bTD-k{-oV!4d=ccd&~tYHDh#HKtMa zK3lP|XqU%J^Vn1q{1+{tT5u4X`P}-^@9MEAIIU)zASyoE79>$u~dM_tQR_fxL3L>G5Li-fAn)N$ZU^C>1u3&2)Ry z{$h!vmg&(3T4pBO>0Zw+pDY*ADq63T0Xj^z#$d!)DbvI5EA>w zo&N!)$Z9r>4GlTPFt>Jhb3mHJ;L^vAZp)PzYr9@_|trA>c=GamWw6_G04bjlQ93!UkR##rZ zQh@XEZ5p{??5p=OLi#Y?3!(_8Ya}@4{L6CvBWt}O4VqtP^GT|56cCSonUn4t8bp%P|KhX%t(pH<4T3YTqB=@{Xomt({ymrC+1?ga z79{Ge`E+5~dyuG0pXjjO)D?IPNK}dsC;#H}{|8b3eR(WP|GzT+r8^?{S7*~D8az(| z;?Y;o54*mNRjOC4)|ti51%t3^iG}fY&x7|5kf<0`%jno3gHN(Kr1WnU#eWm#ix5Zw zUkh2JCBbFE^p40s(h2O!{2SZi-}M|s@Q>aVC`nG*ipt!cFB=lCtJ+NFTfkduD99cp z&PIz5=xE4vFd@;W`JDgX-W5}D|3H5Hp7*dA?91rJpr3ak4sHD}Q2gW1FLEFmU^iI4 zw}KSliqiJxyi=i$zOFp`!ydKYpb4mkHgf#OtNz72^J#ePA1F`ZXrcl24>;7qNWS4E zKzi-S0Dbq_;Z{u*G_A0-lpp_t0P?SMGc?@unrBu*vie8*`ETnC(teN?=!xuRkBwpn zP?MRm{V$fckSCZ5@>8h0NO;n-pim=O-OuO^YYIZ&I;gY+`NIGBDfHD^y^j%p6L@Gr zm7r0*hc-Uv2MOYivHXv{_TMJ@)t(){YSj&)SN;y9)to*<<+ru^+S7KD!s$vg048)< z10^Z+Z@B!I`a%CCiUfV%$N$#l@A%?B2w?lFRhbb)UXURFp>+W*Q}FUtS5KB`9OQ;z z#8%1C=x^$3$Q$&1qs*wByq}=aNlI4Yy)^+~Q14#fx)3DhFAhQ?(`Od!x7QN+t0#ja zVns**y}sqcTbcrg?+FJz0P56Z)=>bJB--HA-qcj)>lOpl{?}YnG6W{{y&mkM;4LFf z^>2(|IVr*Pw>?@W_`RG?VI!F;u@VZZL85AGY51?s{SEub81_=<#K@{?j7J~n= zCb86@sTxwUPp~q=7NpYh+Ijo?L};K#g(f$udO>f`Qg!D|ddtLQ3I>Dxvt0JF5f3Zq z?M@HX*Kf_0pv9|~&^Sh)&NBid&QFQ>@useZURxlIN;@U1n?OCA`TQ68Z)-If^k|mL z*`7YAy;Kg7H(lcG$$IVdq0&43m8#VOYG`?;>X@fQ-dK9Hj(Rd@_k0G9bN=BBUwghq z`cZ(8Carz4YA$h6~AZ?p_`yv1hY=j7a7z_-6kR8rAD+mq=|nGR6u4hqNrt2DGb% z;tL#8#RZtV4@x&YBwKymnGt>9C!7EKEP%X?aj7@7^uIT6pd?KGQAFkXx(27?y%L<{ zel@L^mCI$@wRTPEs>BQU+JCT?nIHtb?=hNk^-+ZkAt(ONdTM`W|m!aO_97BA1fjd7gNoO*uep09Wj$ zw6l5Gi9~Fwc)EqPsjiG%Bkzo5j{g1NENz^Enncbsd4h?IQ=J_W`38$^_#Q1Irr3pt zilu(s!Pl*h%%Esi5UX?xkKbE(Z5s@I3-^pyu-U4jTy5nh4XwxVcwnrn3oyT)vWZMK zZxOx02^`;eu|&0d z|I^l2_#_YSIh-mW^LeIa*<$O?bDQK$;Vw-hwEg2%(M6ghQ5}%0j^uCsM!l||Uu{xa z?}K(g9Kj4e%-U%TYX2r_OYR)8L%#ea%rg#T7`%e+$Pfo@7YIlwwT9}?ghz_e_|IeC za-V8V@jJp9S#3S)#U;!HhX;q5N~V>YJjl=b&aT`wz53C}OBPjIcs^b&OqM31c}?3o zW0$vkq(;l-rs1o)Sd)g{_9PgcPxLOte;cm%(?AO%@xg;D4nO9CPLVQ_hz8~&odC&7 z4qe+c=Zo5^;gK`l{7!C1>R}pOKD0U4wlczIdG;Oa!*3brB^YD1>qzHw&3Q88t6=6_ zT;2!+{{)0@wgyv>$cooO{b;F@^#`<90> zcP?4MH8}6_S||haqN&7Ew`{L(p7u1`1E1AZspf@NvY*nhDCzVnM5Wg3_2zYb|F*zb zAbZF3^VFjUGMmjCH4K3QW(k(Ox$+l;WbU*>>C|@|3$C#Kyim>1?Wg6mvTz72xr*j; z1M?b4)O&Ph?@C}8f5SyDoku)JcsHiCBf}XoL;w7Tkp^KT<6Ql_le@2aV4v6cd)-ye z$B^k_-Sk@Q7-wS4U8{DJ8H_!gxuLqKBX73dKkc#iQVc4=5slnHqrTIcM;1SHRa76X zdaU06O8qQ+#%@H{-AvxGs`c)u$vey@x4r#is~HCVG6vQYlJ8)1K>PHWcBGa#`d}nDqL!rsW`djN@UMbTQAq<^z#x+S{>xgqT%3e4 zaR&x`z|LP@AqX90!1VL-HZoHuu>==lpWd%72vFubKFoT8(U;WI_;C9(dS@P5AlZZc z*@M}fa9vcF=T;SUa7Y8}iYOL-bH%>I zlMzF(&tA3|7g|y}2)(6l;^{KG(KE){|2_3S3$LdBdXdgp63_C#-Oo{fj_tu^Q6k$c zy!P|*d7|^QA(>7j-XGV*j3x%NfG9j4uzd@4PFKjj3a4>=xn`L7aRNUyVG~LDVS*tS z^0g{_iT#b;g=heBb`G1Tvs#QQMrE>H;WZeYmxs4Gfrq68H&BVQQ*$2Ky+!xKz0@V- zcSYvyG?mG!AkseH?Ak(MThSt@I1@B{2L}_1fMt$B9uLWi!H}rcK~AkjwIh!lM(&av zpjkx6NuCEC1P_Qv&XmARYy{~f1yzIb;~iQ!4xn=C=Kp!0ub=?qip+0+}a`1T7K?` zwX`1G0VBEvdvPiHc^u9fThNA&*v(-ax~16$JKp^l6qi5~e7c>1$)(JpUpeTwrqqG{ zCJq+cfj`{O05PbVgC&?Ni+rXbo|cc2ED3tOFy@uxX@O40cE?J~i&-$P05iXm5YG8J zqyD<)i<7-EHs_lawy`K&uIP+!?XLLTn?7xPHitSfnm;(6RPs?=0Q_~k?{{jhkROBY7A)h53jMN$p)4%ia%uB~AkB7KIgZ!OQQ$bOciV4PmC2fmKmI)n12#4` z=`lBNS)iqBQxE;x0-$Ec$V4xXv!z)-M77}{OG{GA4Y3mfz$TQvJ>6|a(Q83!tx)cO z(UKDESW><02=GX>GjMZgYOZcz7qXMbybRx}m5U6M9y@fw=3lDNM3`7Cuf9k&yPH2) zO~!D`!J1EJ2IlBkbv9oZ>jif%J0c-=tl%eavS`?EK52nVyWT`KKKls2X!~&?>>3OP zwnb?VPK9?*L#ujX65PwRA)Mzqgwut79iKt3o?j3ztXPRzycCHu%i0`G2P!-qE+%-l zg+>-^Fq+R|;vcH7r7XI*tPKK6-6O`;Dn$YKuI&aq!vG6(dnUn>3;wb@{6JmLR^GjOESa*S?t1I&3Pi~aJL!`v& zd!{i!IzNa}0X-fr2j=}J;#HJhJ)U0)(y>Ad`#`_M9nV$aa>C*v zKq;Z!d}+8fIU-_hxF>Z)%0i*7`oIy3Xzj|w?;%A*ax-HiiVKBSVA6|lrn!;c_2xKS z16Rvve`A~r?;WKmOIb?Aq#p_Ez-oeaH;^GXq$xV2Y0T z3<12$)`v*$a2GGDedmOlb0!Vf$vkyS2FLxvmEUdLq@H`8M`BlFn7l09LK;iR<)-uA z_dVRBGP_>WrMl~bvBa(ivKl3|>DNo?{0W9KeNiQtutemq9-n&;&byv#J9*(MJ(!x0XL{=a$aGrMOYR%=zEUKaw;M9GYR(n64V(@%V zje#Fjp3^lcsxILY$f4{0B33!&EWL-Y>7H;s#DZvjxGz}8*<*sO~x-qA-7@UN7=dpfbI z1lK+v4lmcw+8)tK9hI$yAh!^&ImW~Rxz{EG=(X@`#>s+GncoZNmoej+o?Z8Wt3KRE zPL$`gq?x)-gJ}tpT3&xce9`d;EZb~mYiL`+(#pT;8y;wBM!HGq1(w!ukhhAKUtYIP z!T^fRajCAJo;NgaYnp*TSCH}QAWJK#iZv3K|0@2duy8a2Y9SFPu z&iMQ93}~(pA}rqYN?2ou;g&yG=P!FXRmA9&_H~>XKs=`<$kp=gj1rpMYxFE~A>O2F z`7~zym8|he`~dR5`N_0(2)Oj%{t4Cj8`Y_@2DsDtmsin;o4}%OOa8E}6VTeTIIo8# zTEP3hy7Gq{&pj*5vt7h>2a3Z!YO)C_A9}L$K%CcTk8Yo=`Xw-Bm-DKi(Lf{pU7Qi| z_5O=Tar(wmn+u*(`{Hk0FdGkDr|UG`g(mr(cF$qHdWUNqO}CIX(qWsLeam^wIbD4i zkAik(4=GCZldoN+qqq}OGw(M$nu0x;4L>sxo%l22dxzl$uCtJ5WG5i(QC+3#7Zya< zj2gop>q=y?vU@U*;oZV3;{IuHuwtJDSSA0|U0{A3Mfz%PXolAw+`ISf)bvv+Q3-bmGB7j0bBbVK)>HX(CQq*+X_5LM84-0 zW{Gs*=4;1c3^+~0cF+!B!Qo*I6RGmb@b+ISSTET{^1>Z}GqHEd6dU&WyjB7SH<(yw z%x2X}(!*5E?461JO|*^dWe*!}x8QQ)w6sQ9rMqj(yf|9cp?TQ9b; zJsXTA9jlNw;$>1eYT-)9;5!0pYS|KV;zp+rcCiJQNO3o9(qH$yjhZP)dOw2Mc%50| z{7{CA2txv7>Cp+7N4fjowKYMy*L3c18K!|zL;#r3n3p}1-4ZD?1*W>Kj>Y|lXhzsJ zTC$p%<8b8s&A}bb5M-~FsPl-ESege*o|S@%@vgEV0dL)?bY}0!8{od_#M*6-n`Leo z(2s(pwI?f$P`5=&KFjMj!2KZNq7`9b(Di$?{bGm*`wQ&J{ZEID`dMY`Kc$kOJ8k$4YkEv#9J;R|W(!usia76s zJ)7`kiw7Z0^p@7Hz!OtZD6{r;OT11cexH<+%T8HGQKygK`}B=932sZ2)8ljWZ{x7c z%F}A0SS)jhVn&|E{(dgFbDbxoJyz#YD_1-02r_C8C<^2EYz%Zr$2l)sA=rOf9G>C2 zyLA|KH!Nj$4+meUv;GQ;UpOW3SnJ@rn&sqr043Claqcw4=aO-jJ~K5_uISR_@d6XD zkF|V9Eo*Qnx&ETnP|nxEkCkL+@3uQVd}vtAGJmOd&w`XtFfzR?o7Bt?XQMUviSjD4 zEF-)>#n5tOEj~D<*p%m8?k1lY`;1^tuglK5H<#rFb4Px#C6lht-Rci^R`^OmPcrPi89jgWrsH?dJ6x6*?9I*~3q|-QD6&zJl>f-^X^&h9-@6P~=)u!p8LTt1+!*l=U+-!~m%UDPvnx$oGAX*E zO$vB3+MbZqMw*tx{_R9i#i%v1`lTLXG#TmB_F=0S@*p3}piP9Tqh>;;Nr(C8{QByb zG;{Q?aOB>RB&4Vq;)0B!mZuM%h^1@7YAK>#wz&gq3X%J$WzWXqg*2V1NcXy9zf)0c zw1(f`e`%Qyz$E}n*jt;?S@(+-^oX@xnCnWyF1&IHfhxvIG zTvdM$vYf-Iif&GGIH5+w^&ZUE;fR+8pXVT!hy%H^j{{~e1ActfH@_Zn@`;p3I$sFK zDiwY(q5Au9TTCr`ehVu;{)#!i=>=*#vsAtSGry(VK5CM-p*IVB=h4w%&31=4){4fC zN`(?Z_@gH05G65}Ja>qpYiK@8Ud*C+7T*)E4hPbY5%cFh*h?4r{hnxlvGz^%kgk|O zEf4kZ)NpE#l;+)=gc^)c;f&w_u+SogbAP*^yje%$%TLpsWs&UdXSYreh~4)ok~RiS z91VIZ3rooWtY8uuhEDmLK2V^&Z& zFPVb-qL~nM*uL{Py0-ZlFNSt4js9{iP5r*FXe9jEjlbz8BfA}RM8Tf9(IdU&!31R? zH=Y(Bav7o4PI34}dl8G}f_qZ2@NaVSk5a99ttYOM1e`kWQm4LSZVmsH>a?q&lcCRn zZjt30ZbBkGaAbG4ZbY8D zwzXdl4^!K?VD}m{T%iE_$E2Y(XiC6{Q1?RtEtGUeFU#mSPSQ?)J$snc@x?hRqB2mq z2zc7a@)X0cP(Wr-`yE#>vgn8F@Qe_tp&9WmBODCq*ad`R!f)*n%}69d_ssG~!XP0pT*&rB~8fdrH3l^OL`>CXgF zlgNtJ$C=Bd`fN4?bu>3$#a$P*5Rw%uc3GL5{M^bQ0@>MOvYt{?sT1oRqj)-s<$uGV zGPJqCYR6s|sc5C21GY$am7p}^$w&cRbI{ST(qNq~FVe9^cgyXUr6&TG$+QMq{K*R# z5J=z}yGq*Q!&iT>U2M-$F+%Z!;0|ii0^EMrX^OP$rec!6{9Wo6Tz)+*l|vf)2_1z0 zt}kl5cO_IByp1lkI}PP_mLsbitaM?j+#%F3lSPIqJRnCX$nC)&FGDChp0%qP5>BlM zlki{b`fh(AHY>!)m4qDj&lqqoJ&jX|$Q;@nXjIiaY-M5C-ceZ)IwX52Gq;%7#;^Ed zn0tZe65SXgRbuGO>Aq2%HZ<*LY;rvFNzFjF;TQ+v7u8N6PgLOfm?mBU?RtI+J;nuk z{mE6+A@_c$)we70s3@pmBvOQ$&~N<7#VAu>Q;MhUwvLk`p!o_ohqvw`wA4B26q#1a zGknF7tW@I=YQtn9o9(Jx&zDtYWnGTE)a$!V;h3&ip%Qben`-rFFq3@0Xk`ecRP)hU zlHuoU<{2oZu^Tr)`b<3+UovsglHWbKMQBcyKOygDSIjrWPX!z6*pE|w+i^O^+*4F3fs!aO_5LC(e4&Dzr^_dSU;FXVH$9x$#pz<2#Q+6vX=}CLPe#>hnaR)b14a*8mls zU*vu@zj3#ChIo(rve3^jpG?_ozV8Li!($5JhkG)?mCZVS-Evz;z{b{Xfxufc`m@<> z2V+V&@U9rWycl;TyIS2gl@V?(EK)a(Tc>d`eh18N_lqFrWgYDOe*48!gy3nO*(9+U zqS8Hb0LW9_e)_e0n!zC>R85?eUOzTw>BvpC2IEsVVkBkC1f4MskgYZ0Tq$Oq{{6dE zi@O;V9Hx%`4mG9^q*+6lvgKEJ2b=!TnJv7A+rqR};)rJA5ElhM1l;h4-pozCFDeKv z*k9GyDkzh*u?2o#I7rJM2K0?D<>&kc^FZxP+@0jpFq|}+Un32lF0i=@Tin=e`;@8? z4fN#)l`0;Xd^dz#lV+l6%!Kx`_jQ#9Py=Z@~6y%9~8F zn6{?&mD1_xnP*OaACBd`Q1?jgVUJ%U9vQ0fk_b%b+vyjXXOw@}k!(pp35VrRMwcqN zH4v~+UT8$eIWRf!5(osBve|m;ve#Oi>b;_$f&SfjM&X<_S zdJ(0hP3lFLLQL`pO+j*g;r<2vHE@Aqo2fNrFD1XMYJWNQlT^r&1lVT}q4^r7LMi3V z6kic)e|tqyww7kBhjXn5{BVheT5A(vSHX^*cwpWy*dhkDM*eqD5)SHI1?O!b`0o+_E_zxHNkd~H9PLa^E*PAvUh~)K6KnqSF$N76|@gPAvkS^ zD3{wfewJxKR=i0EQ23Z=d^ok8YgDj5aTrRi-Y}(9L!YZEU08I#r3%_*EZv=2EKhyU zwH}g_Ph`q7KrS%n*NLEccN4^wYeaXCkELzxHJa^$_Rbn9J|$y!tW0?nMAP?R!^H?! z7Nj+k^t$Nqr{_Ih#C)iFoU{v&V&Cvq8RQo3JglGB_=yU_z;o+Y_w;hY;9?4#Bc~4b$a8ElBGa1;?u{>l!%CRlPbTWQ%d+a~zr<>b%8|F+ z2tyn1tv9QcIU?Jj)lm>1*H z*UWAmLJ?-zc4l5-hom)+kNSvfPigMGZy*o$$Mu+M-&JFRugm6cN1O6i568ZtMin4yAc9I+Wj`>Sk`fyC?qf}!Rm-^~5^wnVj$#(HUIdysL zxbW{!DS;R2l>~en)H zrphsHqD;A3)A@E&54wd=R-w}!)2tu*%wtcQrSKE|5EUsQhFkh2G7XYW_-D%>3NY;< zQS+k1%_)DZK^;e{I@`@pzjF1VW~HXd#hGpp;C684(diDU9RZfmL2cs!H4Sh z;h6cWS;~sqU+iKPt8?;0IfRD#G2B{_yg$;Y5%%X0t{V|czwrS5yduR1Ldtx`Eq zWqwEpu#g5!6KjqJ$|#N*gN;}>smTogIu|7f1ynWN$iX+r#>m4LbbcAIYXO*kS5j4j zLaFh!e9}WOKBj+G5(Q|jKmleNsv!??3Ka)si z!I~f?1MBC0=Vi?GEB@>d5VospSPi{GBoPC(3~&QZxCdIz zIjL&v4wRI5nBs&XERu(@%)mO!W7y=t{@e(c*b0j^%Upo<#z)l3sPPaES4)FK+KaWF zwZvqM3U_NWx&Sn`UCu*LnDmxmOD{u>&j>8*_2Mde(FB|dhGQ+hfg)HpbsayREpV8_ z_heJY;jOb9GhT4bbGY_ z3>VrL{?2?#gp~4Bc4$)LgN&wndb2>=g(o4Crx~0&m}Y(^fCF^08M)lkI}a;mqN3fL zQN+Xds`w_WCP%6q^Zn0*)-V0Z1=+tesyj4kE(l&+u)|TV@0n8by>QieAd!juE5cH@ zc|F0fWfA+79IL)~Ksf-p!*xQb0rOMG2$vLS_AmYFS_lS)tFDLNrEBbxrX{1PBjyHT z#hlW_%`8jLF9bF|^J%QqsW*2zj~Di9KnY%%^ZDCpLAh9bop)pU4$ax8uU>#AFkQFr zVX^u{2}T7c@Y}dhhFAfuBE;|9GIrg;AM>N?giuV z5CZ!iXrkmhJDufOVyw<8c(}K$%Tv!ow^KPU(2D!z2?=WG?uP8samM!r+Ueg8dp|nHK|K=UUwK*x>Ut zv!wU44YQXJ%kbYhf(2BE&FJqK5%%?G9Q9n&n^#&n#;=Xd?El*7D4VNxDHM0D7iLCr zD7{mU1oItvZ#~auD8neHJEmc=8^G00qU{Ey_6u5waINJax~VC$oL}WQjmueHmIF$S zYF;_AKj6{>ODfqXoSBiXtRhE^d_6t#ATxJopg*fLZOl!#7lv>E2eNS}FhJVNTm=hR zc@0Ml)l^Z{@iC$f*{^w1LG49P%N2KxWO1=D0d#a%LC5*na0On;&ilQGZbv!@Sm4k` zr<46l24ABvMl$|hvx<>feEDX67GU(tR+?Sm^4!vPj9f)|DXGCzkxPZhu9;9h^{rW| zrz_1`@J=n<4CUZx$d~5v(iMX5znrGG<37k|Nsud5#0J9ML@0}9g0`-y)Q}z#WPaRz4jC_>s(hh%1D2vbEaw#s+4~*{SLO-uoX$Az3I+#5(dkie)BiH2?#y~rr?n4 zTbL|%dClB*c~J2l+(^`)bxPB5SiIL`)-nDLRK9_1hH8#Fx#eKOS@ZCcx_J<5Z96k9N_IV}wT@*V+jt9`oD&yK09n#_hrA0!%BEP^Rr= z;?ev9Fh76?C@{LCjF_9yyH=Jcn<6`&sUzYr%G+01*2F{caq zybh4$TPdRAx(+(qw_3R+^U2n^TS@L{_FK7{wLfw>s*Bhjf-mS@xA9w9N9E`Xt_b<% zZT?y4P%n~YGtU-R+$N?6no79}yhXsVz{pCY;>doz1#zV|30 zr){Sj4rbuKdIs=hi;m&49@@`*T1DRA=*0}r@goe~)dJ0}#SdIn{hp6EQDB@#UDlj0 z78eJV<#EEFrv`*}y8~`4V(E4x zQS6q_^#d2a%wl64vI$*U`b0I2T8JzNtw=1pWBC20rFwJ0gjo^nej#W5c{|*(h2X@E z9_wbZp;h>d$C!kr-2$t8YX^hu3UkXDFBpKUb2Zz;Eo+}9#|P@qx85!=-}mz=H{`l&m;L?D*%ZMVx2$jSlI-9z zvkszVH~tVac}QRu!U{nTuudBBNSn^ZV0^zz6ppGd2&D(r$)~B_i+_>xqp-;r59N}2 zl+t@Pf>CAzIWm@Zh?t0b!pe4e>RfEWVBt6yi>P9wPV5sHti`qZ}vMsrv&2rB!8E~Q(M zV3`qDwS82c+#e}(l|p|W+wNx7guPeg%yQkNVjcG&hu`*cpo+C-#DKcAB(NLGss1pc z{SGZB&A$iZhk1Vps}cQ`%)#9Zc|`7qUaOO-GODJ*KRFWGX%CkIr>#If!-uD^lOS2| z6sj>0!wJ#6EGE_}&#ID(#F1N*;9 z=2kn{Zgk&gY$#Wg%g*+vB0A%w-6iKt|ESh)s%(bRQjO6MENu#d_53K?k>z-Gy9jc! zE$3hU*ym8W)71I`qZPq>2@2k6{;b4;$1dYMoGZgaXg?r1LtC}&QM{e*qh!|aOZm2X z8fLFz;eEX%VS|fm6VqT$?brZcT+&f3(1J8v_M#MkV0-P?%=i5BzTrFHJ=}*%+cOPZ zn?U+P>oiD_v>;of`*1zCiRA{zeJS1p$2vvX6HvNEpywMHMMXqvb5`6@NH5zYz6{@^ zn!LM?8}6K0*ae%q>G#W}FNyuqhm(|MbZG5O-~cTHntA18lT+7DSaU!6XK?QrD;Fc@&FpL+{?7WaU7P1i;ap8 z%;sBIE#(mi^2^As=WFuDsr(MZRy|}3^vrM;%WWNQDpbq*~N~^ zgX!+JcNQ3T|Llm(X++Gx`J3uA8ktFfK#CeAk#?Z^a!7L7y6{0!^WLJ%5`eilr*5Fq zmde>U%x$}&hZGrlQ1i$5={ZPR*=Gq4rAE2w7uUX zT{xenWPV_K64>bIxZ98!I3xU}>akh$wVFjUP|PHvvwmBk4Opg4cM_dljxa)YvYvAm z2Qb#YXllJRmU90se+{6}@0}rTC>7RF^1P)TNzl#lVqrB^!R2r{v66*AgjHBg-!`zh zL2&9TzULuG4kUc6keUf@j!M8B?{?Cv%KqRk_81rv;jYM-5TvO+WMIOhj9w)WeB~!O zNJpu5Nz^%bkkj;X4^@uOJxzS?NJT>rBGpAXV`6#+4|O+JmYTwA19WvBuNjvA?4EbX zXWu8oUJ!lPvV|jYEUsdDY#K9FQTZ{So*taR(=UWbUKF@6mtz>l=iKv|iDusk7Xx#Z z;!vDN>G_Yelt-uMxgf|NTYkV+0iSlIT6^kui)8W3JyR^c>CCyfu%+pNmFm|VKqZpYU&P$&auh=~gzxROca1cYR@9aLQvdE_ z2!7upr&YX?hbLisRf#Xwi;S;N%3$LBQkfs}&r_XP+< zlY|*!An<4U&+!BJF3GG~n@IWZ$7d6PHm$kQwRhJ_!Cp))k!L}m8)*R_y> zhs1&qLWlc@6xQDraZ#hUy9S1-){VZLN)&KsRfWM6BR4Vq)DuMIXZR87i*d8bi-gMI z7Z}avnQjVSfnW7G7BKl?Mr&pDs=>)&X1wmn^2l07!~j*KQcXXzW<#PtPIJG-${5~3 z`9#i>_(U!{9Qxg?QG8mY(Co-o3&k8t`PmK=5m`lqVc>J?nsC+*@*s8N?hTNP;0x#b z9iJ)J)SsuZvu8~8mzWzLdR%(Cq6QAlqx;fn7F$zRN`(4Zw}*}rHikxa4yayjrKQj& z-`AwiQv*$#-$(42Y&`J&ED1*(nB;@?kCzkXSMq-%lvLVH@Xcw)IrCF_nkUrp{ALXX z3b%8&Zs?^l==gZuXK76CvQvnZgw%T`QmGW;UvWk2^vKh{qJ#kyg2i-+;uY!^tVhtaG$*^54hSM%Kr7}4U^XJR18CigEJ9GI0 zo2ST^3?4+-^RC}FlQqwP6Z}V-jns;h7RjzC1SA9Hy zU)HlP(3f1l>n&4_)*2kw2`u{1!+tIH4s?Bu!ZIqqaydEDgeQHi%R|ccCesPj3RL7B zh3nJxLJ?Mjj#J_HArljUhNVp)=(^p-#ctJSM$)s=Z`MjQyILk_PZkh%q!LUQWg6f> zM$C6%DAk7T=K4>kQIK#1#W0~xSg1<$g9oHR`tH{gQOPJ4?d)ID0l0ohxW*ex_w4(e zYBPr&Pbpv3ZS$_Csq6LH(7`^{5wSiWW1bLYS5+^pQRp2dK@oTXkf3)N+1vZtRzIOH z#fNqq$>XY$d5~<<=q~#A{g=^!<0v>etzbI2rUs8oum2Nwv-?{!Y7bR5g!nS!5iu+L z?s78U1xsCbt*k6zyQSUynU;#cv-ZOF6v(nm@soJw9G75lAG@G)w@0EX73>_r^`O?1 zQbcxACj+^937s59z{t5ta` zp{t)=&)c>eODi<FEYph78DjBn2WZ;{F-M%vT%V9;y28`eV6$Y%+R21wCu!>dPF( zW5HH=!pF?Qq#wH&!FX<+aTuT0=;+#u6A=8m=cm9zWYi2Z>C-lmL~CIf++pssEYPwb zzd-o^`ugVhI+yM5ra_aoX>2yO?KZY;+qP}nc4H@v-Pmeu?AUpC?(d#+&*^zj^VgI8 znLT@X)>`wOXa2Y+lt&a77*a&{dJQH63Yc&?no1!BC2%C(Q*wiP)8Szjtwz3F9CIjg z?hZR;k)145p=69CAID?^um>upA2AV4LOR%cR>n4PPdxN?0W}-%RBt*?i=Alw6sgE8 zV!!uI=+0AV&Tr%tovA!jIB;)^ZLeY})wd#=u3ga+Ty}!i6l!(dbe}-OZ||=9lY=O{ zFu)|rA9YEJCj49wKXG}tGs-&w+TcurB7b~x{chXMZH}78wNI>pJ5H*m&2q#{anc4gr}L!heQ&F31v@TvUg;gbjYkNQO7ZN2lt1ezLsYqQ)G%*r8q$`vIzXRK8mx;W z=MMMrdNUc`2#>We{bpyhgXg`6PoMkhw<<&O=bCP%7B?t^4ov7;W7jqFHzB)X!JQn@ zca+BZqcgjuh7^#IfRhI75@%FTwHE!$vN^Nc1u`?`HbfY%+;Go}bZdEw@B&=N#lY}(q z5xYGOSJ2AhWxu+N+wpAh9!1~6VjmWft<`z-*fq!na;9o@R%ccuG~qm@%Mrtd^t^Mk z=y~^u0`FuW2ByKdayS39w1nYwQW{C{25^%bq9Uk#J^iH}JQfbV=w_iMkpG1#NWPMU zo>>ATpuemGnh#F15nI241x@?zfNU5r<$uBH=^Kt%&4tWKa=0np=H_$IXIvj#7G$-63ug(Mn{yMcQ#Cl7ea%!N4?Tt2uz$%EbmTQ|x2tAsxZqjMXo2 zhjCIxSJO7(vF|d)*D6Iql~FqtGSRSWb#%PY=$z6-6&y+gRq`YVRdP%ZU0yc02fCX^ zCn5Z(ro-UW2q$OmDYp205d^kMOhy4X8(h`Q*84)Mabp!)EaR@aj6UjQ_0i(?!IF9nN@Q=7F&HO?+Q59#P)`Og)VG%EkhwWncpDVI<<{3cb2)JnnUvK_l72y z3A)TG?ou?(E^Q}ReTBwYA{CDTe#hu~{G ziDJq4;RG&|eHki}WMtO&6nM6(6)QiwT_!Kpcc?oT*f6=NOe5`d`$P4^hqlPqnj;W z)4Cfij;+hzbk@L`xpz<*F*r7n$gS{w{6Tnj6Hry*Nz|lI%Ry<{e2-OmW2pzfD-;We zW0D)<2plrL8T zU@9me=&&k^p3KIksf%7`fh9H+q?&1V5LIE2;@TWi&}lK8z;`(QkZ@5|#{DB8|Mf!Z z|9k!=>oe&n=>k0BqB5{=@I=!T>oCyosxe1d;MVhLS*BZgnsSjdoY@~7jeG2a#e&7r{Gn%6+ zi*hu$Bwk}r>&lPq=QpD%`I%>Pj!oE+YBxn|rd-5-`PIKt@cQii9q5vfVIfc$BDnG& zAw7%8`eV*}q0J>kb~68~C*cJW#NX)Y0uN|57YCIFl;$ObRDjTF=8C*hx+LGD-YFSweuXuwacurE zxH;`KWX>i|!57bmkjq$ScGCG$?o>U?a+;>wvo>$h^OSa6V;Pi0igG)p~7aQpi*W*kxY`2V+T5_5Rr zH*XmGg!#V6{r4vZU&DY6OLg2EFqs7}sRwyV%>#NxwD`^BLMP*BE>zg`$e#mlanbDZ)U~*^Ythikct&3d|ilN>G=i}uvYd@aa{x>2?(%W z9oJr0r@QXNv+V_H3=Hv+^CgXOWZlYU6j1tiy<+`|;X=4}ZwO)HwdPP{1aB}om`^?I zjC&50YSVe{mh$J%HR}d$TQIC&2A;E^`hO4)Gz>kd0(N$eS#IZ|q92Dm+i{$IUEd8Z zmQ2YVc|e&urt>XbuSBtXd~8-5FR^Cm?wi|anH zPJVl=>A6I})A2BhL31B60rZ2CqEgc;<4y#ptDv8^cz&rwDB5n<`Q8ghD)gX)?D{Aj zMZ(nr5`DF*sex~63|>N$8nqHYb>zgwdcm^kb=gBHa|3!K@!U(5i6rhad?r>AkeAa#z8P##9c54ApAiITY!KE@VH zkqbBo3RAuj>nkIASOI7bU#=x-PwnD2NX9C&$}};PV*K6Hco5)#;Uf5ZL!U`%E=cE5 zA-7z-?$ktsb=JMtU6FKU8@-FJQsk1g`tFfHIWeG%AB0Xeu8A0rwdJNi#Q6q7b}t04 zkOz?!DY`7VVMlv{bCl29SqI;P8sStfw6bGng(=2)joN zd3hk&-nu>u!hc`!VkCeblZUVG?E#ij1JY?0OGnVE3ZS}{dtP;mn?Af!ujtPjNtGG5 z(qfX8_2R=0&uqYKuw!0((2TU85?F8EquMjj=ZP|qKUoWgc~J=cl>UI%LX1@T%$a_~ zyqJ^=@>#o)F>p3i$EGEkx_phK><3Pa6{c5|UrEBE zDToF2d%QwZiT6!pbgBofChwyLN^3!GbgulKfA~e5;^dk=nza^knlrwqDj#hvhXFAk zned9D`5tv-)=phr<1FO?cYbd#+88#QmaU8iTRTOR?6{*lF<`3VF~?ZS^iZkKfLr^R zJ%s$XwTj1<2p9%!PudKKTwj~4d#1nyr1JpA`f@~C?y>xBrrR@A$srf-s3Id|I`lgV z@fKWYEUF`~Fb9;}Lw@z>yDPEFTLBogWNV~JYBLFmI!5py@N9a6awl|U+OKc-p3_Io zfV)X`IJ#@?oJA}MJ9G8!hHB2{+!yo)N=~8*nmyG$$A{8ofcwWtUvV|u~hdZr~Z&>SH$&s zOkO$qS~=Ng6_ta(fSxSMsPPe7Q9oC3cfNwdq~z1)&*<)=1h*1O^A2m)iq_`JOJC`* zLmb&3VHJtFzF?{42_Cmni*i^o0#Lgn8Sc?Y#Ydx+JPm@_mGcR(BNa46H?{PRQO!}z__s(eAu=F zcHRkI#?JT(g~b*OLHXudhC%YpjrS|xvhN+4KV=kkC7N&zG!7>`hqDKke9~Hb2JKVH zQkh)wym{ARy(OEBgC`j6elMyJzw zGoBEPz1YLiLx}mC%a!v)AYHG?0_oCn&VP8>rV=hqOWSlhFwh%P)G)6HtZCH8n(Mu%LnaYpTH`Uj!6?g zB>98729VEw=MAFY&aS%a;YX;I9V0qxqd|{{R-<|pl29YcqiCH-1pFLyW+H;{6!gug zy&XN_p<7!{kNJ}hl|6{VJRd!#D=45ImKwH8dlzvedx>TYI(Fe$?zpf})#HuHaX4{j+xx1+W@zJ6eJ;Bs-KqYTFs@W< zscFs9WmDDvQF8x^6f&b2BgmS?G+GOkiQqddS;et)G8%@`QjXFy_w9Mrx|}(2d0ZV(7IY%f@tbU!gV5< zf;we@vbB(G0^)i6l)IhyoJnw){{FpLK`|@Am(|^)N%#fJQM=ixs57#wO>n)z$Bd(- zHIX~>a_<#en!|_h%QBAX$)Ez-6MmnI=A{FC;$6$g$}t#mb_U$`%3w^KQ*kT#7F)+s zKCA*v$!^N~zbZG7Q+%$&h-S8Q;exobL%o2HV^+(lPf+d~Kf`e6q}9dUD-0YrB=KN! zZzM``cKzf!#~H9Opf(mQD-i2EG~m}w{!3O615D=*Ip!{$6iy$LaBO7bTPY)c`>^kC!# zks)N2{Mia5@9T2w`4&;c*E>wN<>HRc6iZB+-l%#Rm9Kb=nJ2e(Y$0<-YIEB6F3^^x z$b?a9h?qy>Est=(D@VRP@7YdjD}z-`8PXml43dxQ7gy<^-H`Omx)MOk@K38FxvKNZ&xRdRa$I0S28qG{&?J(a6C%)f4?CwSzpswv>)iIYzFs)&1 zyxQ7}xF%`p_t;!NFv&o5y{}M zo)EJqjs#Zj$9{ujBZ3mUd!fG82Q*08fJtS-q5CK}Wt=j@`c#H3?h6i!X+5v7Ie&P9 z#g8$jq809g&BQo07IhejCBvFMdGHPJYQqI9L^?cCS331z&s;hBH?3_DW937FC>QDV z33os2BN>?ZYm^fl!msN%%8)c-H!iveS)D8fpQIPylLFCfYzprEUgkkDj}`QZ2gswD z`q}tx>{3TM838-EG%B+JaI=e1jjih^;~s);XG0vJ!0o)d7PZf^0QA% za5-9N=A=qFI;X14ybfOjxL5jKB5%fiC)laKK?dp#37{wvq zt$DoVuM~77fk96UT=sU)AKmEM9>b8X9%#>98XPgJoSXT9FwlOev9-2{gVj!?@7ah{ znz0n*zC#*wBz?QEWo4zFTCM2dKjz|uSpD&aVWUL!UOQq2CTGN5BCaVqyvf4f#V{-p zYnEA^^2GajmtctbW!j;dz%c?b_=((`e4Cj|)+Ova0luMYksdhAJSrK_DVI7>zzv8}mLo)55kZsKa(Px|B{h8^DS z(GA9+JxcVSb;_VvRzIIinH=zGKCTZt5#$iy#Ayb#9*IihKdmxxVj0`K&(rD0TOh@j z&)Qjz4Gi|x|0pt7#1totA;x+nz=&It6b?z&QrG`sDJ>&i8o$D}+2Pb6%-)b~Q?ECV zmhR?j1T7M&V0cUMWXA@4*?rKrD1;9?O7=gzZ$z;LijD2!?k@^oK3F^^ltI(&ta?6a zN;fN~V@qW+h$R40bS((DUEkOe$MG!@Rs1j*Bn>Uq6YJC!BeZ$j$;G6HE!Kg*Ri)YFbD*>HQMMQq=| z&54$P9MKloi05a0#KBQL~4*Y96@$S@|q8E=(DHI*9be>mZO}ZG- z4?XOCx%Jf%l_6!DF6gVVzjT9mcwdW9DW4IXBw&^9EuC-mRUBkeUE=M?UYKxX84S69 zbzmsGOOd*Rx-c;@NJ87Vro)FsR1x_WpCbZ=r(0{YV3OJu_ACC*>K8&WwQuxl?IgO1 z!F01MfGpDK>ESQVIRwY24oH_7BK2B51;4FLO2Dhm2j@r%9@|xs`Eh$MKIaA!CQ^BUNx#h zZDaw7ZoL(HWrzNodPQ944}_=Eo5sXquG(DH@ukIHn=2GziBqG?ToM7nK6G>0S#~F9 zb4#%DBO>NHZpv~gr~4^G2|pZsh$-^93c_r9f^re-W(6gp9%*cRdYIlxm^062n&4|w zBUX~|zyyk$tffk7S@aAf(<(`*haH;c)uYW*?UT10U(~K#uctVcr+p?)=-Oob4B7=K zR|4E)cRRYz9^akFQf{0a2EsXN)=z<`tSv4Ydq<@z8SAvk>O)pOj9jUx(TTfC^u_58 zLO~rqlufbOuo?TONzf%N?q;tv!Xis)>Q3E<#~_Gk=jz>3TG`oFr8Pof*x3;}i@Q4) z`)t-J$ME%Mu6Sitq7D759$}bFejD2LA}@WLYU*oDP}$Ln&bxFP)ByrQ+%wKXuVZwC z-%&MnwWegPdATEoh(R2_s5O6SNRb;h1Gf5WBajFFHID!bqQFB1<`F*DBFAzL-~$JQ zHybMpFk12JCHU%bX`$UBKZNj097E6R8OmBmu@(J{K})4*vZtkjmJs!bnL=U+T0RkoW2ft8i$rC?jNcP%J9O6%FZmhU4(F2hj)1M3?ggdEB(-rq_7?oCYH;bS&+gH?i(FRGGfP6T^N%NA}6hA8yu+ zdXA)a;NBT!slD=Dj*rP&b+K*Z*l_NY6VV;#aQTBvcU%IL-NuZ%;oA3+Z4GgI4LRChb8% zm4AGS^O=Eg#*rU+YtRZxnQC8JShJ|zhJ^Lafo;Gx%=ALhvvOFUJw7iUdO+A05FHCK zd#}}}4@GQ;p@0XzSB4PotrgpOxBccle?jiKlBFc9zS9UpN^89h{>))OiqZu&n<1`< zF|C=%PxGY{XB_1FU^40g7jD7fa7dJkjc(EH;3>|cfgrW~I^lt@#j%inZzLH%qZIm&P9SI8qd2lf1lC?O-Q~|7HhNX02x_)5G3jaQ_jyH*h%&D5NH0$) zavl!wGn&&mJ93mIV{Q>c9Y?(tPuh-On&G7^BQw*!OtR!fRibw!Q3YJCbQx&wjcvt! zPv=!5CW|x0aBD^}#jmh<)N>p{YNT9*LM)}e1dv=sN`aX2Mu21{1RSiWQlgM0d}Vr7 z_zY0tqJ&T!5)95)>ATXsWikwc>XSw@7!82Q1-(nQ;M_Xp(@b}GUu_G5AG|6Kp1L=6 zzB%KZ;dO_26}DwlF!zl?=vQerrTj+SM}hcNA4h1nCAJ^{P(H}u@Kd^5gVUSSqaW^k9T1nom=D1rRhhN617Ijg!N)sjMESjZsyS8O>rhLR`)N5_d*zM+0F36 zIoc**;E%dXr!t{xwtj4|yNOL0`=t*ml%@T;*^J7)BjE+~JgDy?3wf*WVOJWiC}VeZ z-5iMbp7YDI>bXQP)SZ2<)1zi;bj98>&QzN|tPc+CQ?>Nh(H)C;mP1mCr+}rX9b*{D zTtHdMo;fuvjj?3SP&BA$j1FWquo`~-Oax=qZ%#8*{8}-(cDJc6XapapYSdE-l~Tsf zCBk-{h+aAx5A0<`gaO-lpTC&%;$VGm7y4$b4plU_U-4V*B?K5m-O$;Bw5Dv4)t zopcgOD(POcDGb=jX2(8xG?Zj`S2b=yXqV3~RS$#`zD!gE&z*F^o?1P!fB__Eox^g5 zn}Ln@V&#*0>XT6F?1O3tcBE|6STz!W`JIs2hWcF>!m7z07h-+B2hrtn;V^jR0#5Gc z(yAhBG2Mc4T^Epv^ET!EO>vyiW^%HcR-RV5jaOge*q%IBzK|jObXNSjCigF2>FS1D ztiFZvFh>={w+N$R^R9lELASA7+r;orKeX=6u>q0BujKUI-cCbx#d|mhbT% z2R!KHQ_il`;}Kw$Bn|c&P=2a(EqgtEgu*QJq01tQt~Y%Q566)!!;OJIsuJW&BFx;+ zCi%MxuVR6PLO$geAthx?>fyo_u0D{KG!+eTKtW5S@FBCn+Xxr(@+%hjb`gG8f2KJG zm!C?4bT^8lD-kkgkppA>10HSA`c?5tAKsLbgV=*7y~a_cDU@c##)j#`IvQ9l8LFRy1avOLl%IDKfd^f??E}+s4KFjP9v)BYEI-W-3z`2Q{=H) zN3sN{t~~<@_PeNCYh18sLmN8oZf;wq?uV{2BJ?{-n}ZaJW7JgU$gFCMd#yUXWkt)% zA%a6)Ri-(GxtGp+qZDk|t!u84y%1+I#}U9avPYK6{ZoD7JS#U;iI-(iD|$B=G$iB0 zP>=ji>>13DDXpjo^bL3oxNcq4qHi0=Y#^+dWzKQ_-u4H97y1NDoMO78^TA%RSZ{<# zGkY{as!K1m=t4k+8n_$p|cAl z4x+pgu*Ce+3V5o?pTZCDU-kf~1V2r;;`}nct|V3_d;2d*_~R@ga(EoiW|;PYxHfSa z+kq!+r4w;=*I;pVd&epf4h>Mk>F0@R8r17Bsf=g(jIJ_^m0?&3~}koNZ<2# z*(xhR76aZg_3P|K6*p4yJ1b1P|lAc9<(S=V!yTkgJ{$xWAT zMMv%`O)sU9P|I7SF_jXm+APe?0&EjD6NcqX9tsU!Fi7HWNzC;>eL_!-JhG!ya_u@d zF-^v=B*sujq~wxR=~yRe)PV_^3r=lf!lF68Gvx4KzZ_}(6`kN6@ValO{g}iK^15uk zA=i(+C+E4;>W~KTJGaTi&FxfJiZAFYdKhnu?3A^f(;Qx(Q5lv}06;kvA9+scv5mG( z^bbOG$u+kMYz%{sN;qNf5+auO{ZVK8<@f@>ogTxFxIbAjq>6Xu4({H(uB5wNLh_gG zsOlonxuI^xYZx`iVgd>K3Ivz5OcbPjr4RAaM2}VU^qDT|yz>N|nsewUZeC#^xo69M zpB;K{gB`8bM#2M331mSs?h>yWEw3GNdA3Uo>KaQXxU)A;e@6E+2S9V4e^nGZd{6Ug zGZTj&)=ZDW-=+6H4NW=RkQ+v7%YUiTH=0m?zi(gS@WUm09BJ8pZK-$iCMPUba(t!r z+J?WETBJ??j@P^*$mF`RKM#BfKWvKTu4y+KVSlYiMwaaV8j{u9@>L{EBN=ekgJ4cg zAfb$_;e7+#&1o*4{%cDM*Ns;ih->&&BDIpE*~~P?KW2q6Ci9^~IhTG?v8x2FDpcB4 z>BrgBvRYuMF52qrp>aMLhwbS~WLZ6xzz~VxpyG&=R%W)p(G%mu*DbMSQNCS^1lH4L zW^{-<2f;TLE}VhZ)Wl#r5)}58vIbq_+HvBVemAXQKG~o#4FoaFSa)L-ec{bw5!M#E z*+UJaoBE1vs5;%9tN8ItPJ@r>YJSURE$dr9clb$EP?X;--SpaW4~C}APWFc|hZ=Fa zcHA882AsX0G!Wh-PCRjeNmTRY|8O2{j9wtWU&@v*thao*;Xde_AGQOke#F zdKlSWeSvU$MBP$lGD;#_f?eD!m^sY9f5XbPtcfJOD4=gc7us&Uw%?K#-mp^n<$3;j;1*r# zZugkPb(o<*$n4EUt@jQ#|2FMImM^MTL_!C?GXrR7>s)+iv&}f!SP{#;WE8{1^iKMZ zF}H*O)xPkq@U><{O%I1RrlJiSMR!9pFxOMW2Yw?#P`cKeNa=7TTwyoNmOTYFvZTA^ zx3wX&7MJx(ku5jKeV`L{SWJ16HQ3Z4 zsL`%iPcT&RVZ7*#HzvH>KQQ_{*KE&?nzPT zTIPI;Dod^}rhFJtG?3EMJca9x^)Tg=-2p;Y07&))_6j%De>BN!EFSUgvoEnrXy zYq|!3EV4DS&ib6p|K_W-Q(~(A4Mx!Nm$r%Z)Hdt<9%|`K->y=j%^BS@sjcf%!b3Ai z*HsTRjIyRTTU*0}VIBqEUorz%DRQ2Uo>p5jG5W}rQZu8NN)fC-HJSB&;4h@Nx#)<* z)l1_)RUoLol|OM*Om>+O_z#!$P6P1@Esw z77$>Y0rm}iEI}YJs~KFMugy0)p~QR-OPFYz*wL=K5Tz=c!iHpc&}M#98r`{xV_jEW zB z@g=spR8x^mMVWsgJ(U}5J0BxleNba0$9TmNiCskc8{N_H@)OfJU)&V1M`hX+#zxWt z2*qmwxL@#T>z^Exb++wHZqvaG}f5`Xpe;>aaYtAMX#YPtri@ zS57f2E|CFNb8?C>vM7SoKC!gnwEcWg*P<-5;CRzgm-xvC4G|h>2e*+9VW9 ztC<=oyHU(z*tD@yg$kMFgBH1GKWsPjZ(77B_Vpkk_Vsa7b0&;EsUD`lH@Cu8h@_XaOYcgEDf}ivkC?Q7X&TAC%`hbN z#1O&-)27T9j<9!)^=&O{g4z#Rc{bm@wx?;?r|lWkMGs9MG}tt%TO;S6rBH-|~?{ENS{xtQ~}VVlyBr zuS}gCj09GFK|{S%U(2apt=1w_Aa(r8*}EbZy>CI}ccaMVJ&9%V?s7Y)i1bow+Q|87 zGL17>U3Kj{wWsYzMj4sFYpeZwNtr2|OI10S&_`_R zaxH})#;Uq%D$@(&9PDmHNhs@1;}nLb(_BybZ_6T~nFUH}i#+Y54?;IC;H2G3Sdcy3 zG1%JSUUmVFA3?n|(H(V7dav27NM$+FHTfQ4SnnG!6a>Re2Yk9yhWP56<_;~!8s^R=)okk?n_C)0!a4R^%cr!o+ z0=@0y=5ySU-Sc4ReC-+sxORaM@9a-+RBjtYx~q9qs0c^*x$0F-1InGcodJco1arQo zITkc#~yluSzyoEhw%W>ckQtr#SnN9 zdm(oXb}O<4q7o@iE3u55^JD~t|B2CnX|rUx>>7kyfIa7EH-;3skErj&jR5Na!b# zY`U}6h0)tbefX9^p|#}#P578k%di?ERoJDYNK44QC*6zP^Jh9}8Y{<2vV4;itl6|0 z9*$-e;akp6TQeRXTt3CEMbpkMnx=v(EJF#R49w+Pa(j2pbp!ktKQ*G16730OSlth; z=sV@vMFI$Amt!cK`_uc>=`w%qyLxd0tA@vgGw~SU)m6L25zjH#z4{}RE60kN!;n&YA@xDK}5Do_><5h+{#FhPQ=`u zu(PsQteK6J0);yD3B&%uDb^eASvhtIB54;o*Sz;CN*%st02jcEQ_M+_J& zhNWxI2v}sZ#<8zDW0G88nPl2gbWf8DoE)#S4Y_h1=^Hi%(H(K%OSz<+q~;Gmb7M2$ zl8(%Tdk$%7jC!v=PHWZ9()$%q_LLc_m*zoOwMGrifi2f$0$t|61??VD>k=kgx11k5 zOt-m7pI}lFkJDDY=^rASscK83zLRCMhI}|B z<*XZjd_{%+*b?5O=Ohv40D^*A*>xNCE0+2#5P|y{mxRca1*iq|A5nzUnLNGqIt%{j ztL_!=7G^}JefJH5mLl~NmS8yLt51&GO9Y22rr;D=!huz6bAwpM=9yzUg|d=F4%p*I z$l!$T%U4t)Y>-#lIUVR)LsJp8NaF5YrrCQrT!iRfcZ+C%xfj@GkDTyp!3d9A9pr)l zX?#L2_gNVfLZ|0DrrmK&1og|q8;>Sy{CXn`J)I`K;s)4=FhS-}_7Hn*M?|v5)2x<& zje;Z7asI~&Ma%_%=)-!GiZpoJ?2V!j%~&xY5{&><-x^30Hke73WniN~>=a3=#+q(p zkZiEcIl*^(&%#vF$RjttUt{&3G-V0Z+G2&6Rv3C1&@eoUSCcNMrO&$zFH56&sGrNc^f2| zB4>d1DQwm;@fPYL2Y{f02FH~6cjaV30auH0uyIw>cp#o_Hbb)9LGfrQsE*q53KE(8 zdCXPA7XWX?Sh$|$sWOfbm$27}wVGP*^pJBy9wsU^6k#X`8@;V;N- z8GqoiRXmAM&hrU)9^eZxb1u=r7t>N8rZ7wb{%jV_=O;%4F(wNe=AIHb~`*LP%^~ZUf_Eu)?Yh~>sVy_7_Ffz6B1h^^uT>Cz|w%BIEqR`Zo#u9SbXj`+p!Goe+Z8`b`*D`Qz^l zChZ+??zDFUWq80q02TrETfK0-a>(UAb3~XAJe1)08RJjp|M@T;^6Hx*Y%Rs9BA{Po zZ{8gK+VuGA35~xX?SP_&c&q2|J3?3@frPL`ep7Xo2MXpI?)&v`%$$Mv)tU-dO?Ij$ zy~an_qc(msxR&fy)Fmeti+#YswU5f|zY`w8S0J%`MzrR>ue{>Ajr<>|{-475&nLU_ zz}T>hwg%|91}GKoN+%iaMCqy$wi@l%?H>(7$^*&)ePfS5ad* zAC^Y}MZI)m#`^Db^IW`2@8Yn=fE)-;9<4tlVfoEq^4E~Lm<64n{t}x9Yh)|IB}^LZ zUrX}JOVNp*?r%I(+5buKA7ZV2U5aXtp2dr2 zfslEbl`q=AQJD}f5m0A38|%Y*uUTL+gEI2(#K=JPnj$7ri_QV#L00F_-^x7i)f{A6 zeN|$dfCS~O>Z$&Q5EZsp+i`&@lzIS4FZ)Lp$!`=x0Q=fFv3>)vKL^qg#gRY}{P(=+ z?<)p+DsXhTc+$4&tE{EY0)97GCIdK_bd8@`_. The ``xarray`` benchmarking suite is run remotely and the results are -available `here `_. Documenting your code --------------------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 947d22a9152..a9532b19853 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,10 +27,11 @@ What's New .. _whats-new.0.10.2: -v0.10.2 (unreleased) --------------------- +v0.10.2 (12 March 2018) +----------------------- -The minor release includes a number of bug-fixes and backwards compatible enhancements. +The minor release includes a number of bug-fixes and enhancements, along with +one possibly **backwards incompatible change**. Backwards incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -44,9 +45,6 @@ Backwards incompatible changes .. _ufunc methods: https://docs.scipy.org/doc/numpy/reference/ufuncs.html#methods -Documentation -~~~~~~~~~~~~~ - Enhancements ~~~~~~~~~~~~ diff --git a/xarray/core/arithmetic.py b/xarray/core/arithmetic.py index 3988d1abe2e..a3bb135af24 100644 --- a/xarray/core/arithmetic.py +++ b/xarray/core/arithmetic.py @@ -36,14 +36,18 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): if ufunc.signature is not None: raise NotImplementedError( '{} not supported: xarray objects do not directly implement ' - 'generalized ufuncs. Instead, use xarray.apply_ufunc.' + 'generalized ufuncs. Instead, use xarray.apply_ufunc or ' + 'explicitly convert to xarray objects to NumPy arrays ' + '(e.g., with `.values`).' .format(ufunc)) if method != '__call__': # TODO: support other methods, e.g., reduce and accumulate. raise NotImplementedError( '{} method for ufunc {} is not implemented on xarray objects, ' - 'which currently only support the __call__ method.' + 'which currently only support the __call__ method. As an ' + 'alternative, consider explicitly converting xarray objects ' + 'to NumPy arrays (e.g., with `.values`).' .format(method, ufunc)) if any(isinstance(o, SupportsArithmetic) for o in out): @@ -51,7 +55,9 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): # will be necessary to use NDArrayOperatorsMixin. raise NotImplementedError( 'xarray objects are not yet supported in the `out` argument ' - 'for ufuncs.') + 'for ufuncs. As an alternative, consider explicitly ' + 'converting xarray objects to NumPy arrays (e.g., with ' + '`.values`).') join = dataset_join = OPTIONS['arithmetic_join'] diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index 522206f72b0..92d717b9f62 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -277,8 +277,8 @@ class Indexes(Mapping, formatting.ReprMixin): def __init__(self, variables, sizes): """Not for public consumption. - Arguments - --------- + Parameters + ---------- variables : OrderedDict[Any, Variable] Reference to OrderedDict holding variable objects. Should be the same dictionary used by the source object. diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index efa8b7d7a5e..9ff631e7cfc 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -363,6 +363,7 @@ def name(self, value): @property def variable(self): + """Low level interface to the Variable object for this DataArray.""" return self._variable @property diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 03bc8fd6325..e960d433f98 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -399,8 +399,12 @@ def load_store(cls, store, decoder=None): @property def variables(self): - """Frozen dictionary of xarray.Variable objects constituting this - dataset's data + """Low level interface to Dataset contents as dict of Variable objects. + + This ordered dictionary is frozen to prevent mutation that could + violate Dataset invariants. It contains all variable objects + constituting the Dataset, including both data variables and + coordinates. """ return Frozen(self._variables) @@ -2775,8 +2779,8 @@ def to_dask_dataframe(self, dim_order=None, set_index=False): The dimensions, coordinates and data variables in this dataset form the columns of the DataFrame. - Arguments - --------- + Parameters + ---------- dim_order : list, optional Hierarchical dimension order for the resulting dataframe. All arrays are transposed to this order and then written out as flat diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index bd16618d766..f7477a3e6b2 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -775,8 +775,8 @@ def _decompose_vectorized_indexer(indexer, shape, indexing_support): backend_indexer: OuterIndexer or BasicIndexer np_indexers: an ExplicitIndexer (VectorizedIndexer / BasicIndexer) - Note - ---- + Notes + ----- This function is used to realize the vectorized indexing for the backend arrays that only support basic or outer indexing. @@ -846,8 +846,8 @@ def _decompose_outer_indexer(indexer, shape, indexing_support): backend_indexer: OuterIndexer or BasicIndexer np_indexers: an ExplicitIndexer (OuterIndexer / BasicIndexer) - Note - ---- + Notes + ----- This function is used to realize the vectorized indexing for the backend arrays that only support basic or outer indexing. diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 845dcae5473..fb09c9e0df3 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -195,11 +195,8 @@ def construct(self, window_dim, stride=1, fill_value=dtypes.NA): Returns ------- - DataArray that is a view of the original array. - - Note - ---- - The return array is not writeable. + DataArray that is a view of the original array. The returned array is + not writeable. Examples -------- From 1481656a323323c38e9923f871d27ecadc26d6ae Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 13 Mar 2018 09:06:05 -0700 Subject: [PATCH 057/282] Release v0.10.2 --- doc/whats-new.rst | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index a9532b19853..882638e711d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,7 +27,7 @@ What's New .. _whats-new.0.10.2: -v0.10.2 (12 March 2018) +v0.10.2 (13 March 2018) ----------------------- The minor release includes a number of bug-fixes and enhancements, along with diff --git a/setup.py b/setup.py index e81d3d2600b..9621941f1de 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ MAJOR = 0 MINOR = 10 -MICRO = 1 -ISRELEASED = False +MICRO = 2 +ISRELEASED = True VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From e1dc51572e971567fd3562db0e9f662e3de80898 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 13 Mar 2018 09:10:26 -0700 Subject: [PATCH 058/282] Revert to dev version for v0.10.2 --- doc/whats-new.rst | 15 +++++++++++++++ setup.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 882638e711d..40e206aaa86 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,21 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ + +.. _whats-new.0.10.3: + +v0.10.3 (unreleased) +-------------------- + +Documentation +~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + .. _whats-new.0.10.2: v0.10.2 (13 March 2018) diff --git a/setup.py b/setup.py index 9621941f1de..d26c5c78dfc 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ MAJOR = 0 MINOR = 10 MICRO = 2 -ISRELEASED = True +ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From 1d0fbe6fe36d5e8a650d416cce85e7994b32e796 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sun, 18 Mar 2018 17:56:26 +0900 Subject: [PATCH 059/282] Improve efficiency of Rolling object construction (#1994) * Make constructing slices lazily. * Additional speedup * Move some lines in DataArrayRolling into __iter__. Added a benchmark for long arrays. * Bugfix in benchmark * remove underscores. --- asv_bench/benchmarks/rolling.py | 24 +++++++++++++++++++++--- doc/whats-new.rst | 4 ++++ xarray/core/rolling.py | 25 ++++++------------------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/asv_bench/benchmarks/rolling.py b/asv_bench/benchmarks/rolling.py index 52814ad3481..3f2a38104de 100644 --- a/asv_bench/benchmarks/rolling.py +++ b/asv_bench/benchmarks/rolling.py @@ -8,27 +8,44 @@ from . import parameterized, randn, requires_dask nx = 3000 +long_nx = 30000000 ny = 2000 nt = 1000 window = 20 +randn_xy = randn((nx, ny), frac_nan=0.1) +randn_xt = randn((nx, nt)) +randn_t = randn((nt, )) +randn_long = randn((long_nx, ), frac_nan=0.1) + class Rolling(object): def setup(self, *args, **kwargs): self.ds = xr.Dataset( - {'var1': (('x', 'y'), randn((nx, ny), frac_nan=0.1)), - 'var2': (('x', 't'), randn((nx, nt))), - 'var3': (('t', ), randn(nt))}, + {'var1': (('x', 'y'), randn_xy), + 'var2': (('x', 't'), randn_xt), + 'var3': (('t', ), randn_t)}, coords={'x': np.arange(nx), 'y': np.linspace(0, 1, ny), 't': pd.date_range('1970-01-01', periods=nt, freq='D'), 'x_coords': ('x', np.linspace(1.1, 2.1, nx))}) + self.da_long = xr.DataArray(randn_long, dims='x', + coords={'x': np.arange(long_nx) * 0.1}) @parameterized(['func', 'center'], (['mean', 'count'], [True, False])) def time_rolling(self, func, center): getattr(self.ds.rolling(x=window, center=center), func)() + @parameterized(['func', 'pandas'], + (['mean', 'count'], [True, False])) + def time_rolling_long(self, func, pandas): + if pandas: + se = self.da_long.to_series() + getattr(se.rolling(window=window), func)() + else: + getattr(self.da_long.rolling(x=window), func)() + @parameterized(['window_', 'min_periods'], ([20, 40], [5, None])) def time_rolling_np(self, window_, min_periods): @@ -47,3 +64,4 @@ def setup(self, *args, **kwargs): requires_dask() super(RollingDask, self).setup(**kwargs) self.ds = self.ds.chunk({'x': 100, 'y': 50, 't': 50}) + self.da_long = self.da_long.chunk({'x': 10000}) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 40e206aaa86..1a252b15fd0 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -37,6 +37,10 @@ Documentation Enhancements ~~~~~~~~~~~~ + - Some speed improvement to construct :py:class:`~xarray.DataArrayRolling` + object (:issue:`1993`) + By `Keisuke Fujii `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index fb09c9e0df3..079c60f35a7 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -151,34 +151,21 @@ def __init__(self, obj, min_periods=None, center=False, **windows): """ super(DataArrayRolling, self).__init__(obj, min_periods=min_periods, center=center, **windows) - self.window_indices = None - self.window_labels = None - self._setup_windows() + self.window_labels = self.obj[self.dim] def __iter__(self): - for (label, indices) in zip(self.window_labels, self.window_indices): - window = self.obj.isel(**{self.dim: indices}) + stops = np.arange(1, len(self.window_labels) + 1) + starts = stops - int(self.window) + starts[:int(self.window)] = 0 + for (label, start, stop) in zip(self.window_labels, starts, stops): + window = self.obj.isel(**{self.dim: slice(start, stop)}) counts = window.count(dim=self.dim) window = window.where(counts >= self._min_periods) yield (label, window) - def _setup_windows(self): - """ - Find the indices and labels for each window - """ - self.window_labels = self.obj[self.dim] - window = int(self.window) - dim_size = self.obj[self.dim].size - - stops = np.arange(dim_size) + 1 - starts = np.maximum(stops - window, 0) - - self.window_indices = [slice(start, stop) - for start, stop in zip(starts, stops)] - def construct(self, window_dim, stride=1, fill_value=dtypes.NA): """ Convert this rolling object to xr.DataArray, From a1fa397c48698b3434e513d480a8a2e8ad04dd58 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Sun, 18 Mar 2018 14:04:06 -0700 Subject: [PATCH 060/282] Fix using xarray's own times for slice indexing (#1998) Fixes GH1240 --- doc/whats-new.rst | 4 ++++ xarray/core/indexing.py | 30 ++++++++++++++++++++++-------- xarray/tests/test_dataarray.py | 16 ++++++++++++++++ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 1a252b15fd0..b28beb9e3b2 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -44,6 +44,10 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed labeled indexing with slice bounds given by xarray objects with + datetime64 or timedelta64 dtypes (:issue:`1240`). + By `Stephan Hoyer `_. + .. _whats-new.0.10.2: v0.10.2 (13 March 2018) diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index f7477a3e6b2..2c1f08379ab 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -48,11 +48,25 @@ def _expand_slice(slice_, size): return np.arange(*slice_.indices(size)) -def _try_get_item(x): - try: - return x.item() - except AttributeError: - return x +def _sanitize_slice_element(x): + from .variable import Variable + from .dataarray import DataArray + + if isinstance(x, (Variable, DataArray)): + x = x.values + + if isinstance(x, np.ndarray): + if x.ndim != 0: + raise ValueError('cannot use non-scalar arrays in a slice for ' + 'xarray indexing: {}'.format(x)) + x = x[()] + + if isinstance(x, np.timedelta64): + # pandas does not support indexing with np.timedelta64 yet: + # https://github.com/pandas-dev/pandas/issues/20393 + x = pd.Timedelta(x) + + return x def _asarray_tuplesafe(values): @@ -119,9 +133,9 @@ def convert_label_indexer(index, label, index_name='', method=None, raise NotImplementedError( 'cannot use ``method`` argument if any indexers are ' 'slice objects') - indexer = index.slice_indexer(_try_get_item(label.start), - _try_get_item(label.stop), - _try_get_item(label.step)) + indexer = index.slice_indexer(_sanitize_slice_element(label.start), + _sanitize_slice_element(label.stop), + _sanitize_slice_element(label.step)) if not isinstance(indexer, slice): # unlike pandas, in xarray we never want to silently convert a # slice indexer into an array indexer diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 059e93fc70c..3fd229cf394 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -779,6 +779,22 @@ def test_sel_dataarray(self): assert 'new_dim' in actual.coords assert_equal(actual['new_dim'].drop('x'), ind['new_dim']) + def test_sel_invalid_slice(self): + array = DataArray(np.arange(10), [('x', np.arange(10))]) + with raises_regex(ValueError, 'cannot use non-scalar arrays'): + array.sel(x=slice(array.x)) + + def test_sel_dataarray_datetime(self): + # regression test for GH1240 + times = pd.date_range('2000-01-01', freq='D', periods=365) + array = DataArray(np.arange(365), [('time', times)]) + result = array.sel(time=slice(array.time[0], array.time[-1])) + assert_equal(result, array) + + array = DataArray(np.arange(365), [('delta', times - times[0])]) + result = array.sel(delta=slice(array.delta[0], array.delta[-1])) + assert_equal(result, array) + def test_sel_no_index(self): array = DataArray(np.arange(10), dims='x') assert_identical(array[0], array.sel(x=0)) From 6456df4e9d103a75231d0ea43bb87250ad8745a6 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Tue, 20 Mar 2018 23:40:11 +1100 Subject: [PATCH 061/282] Starter property-based test suite (#1972) --- .gitignore | 3 ++ .travis.yml | 4 +++ ci/requirements-py36-hypothesis.yml | 27 +++++++++++++++++ properties/README.md | 22 ++++++++++++++ properties/test_encode_decode.py | 46 +++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+) create mode 100644 ci/requirements-py36-hypothesis.yml create mode 100644 properties/README.md create mode 100644 properties/test_encode_decode.py diff --git a/.gitignore b/.gitignore index b573471940d..9069cbebacc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ *.py[cod] __pycache__ +# example caches from Hypothesis +.hypothesis/ + # temp files from docs build doc/auto_gallery doc/example.nc diff --git a/.travis.yml b/.travis.yml index cee21bd87c6..5a9bea81e4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,8 @@ matrix: env: CONDA_ENV=py36-zarr-dev - python: 3.5 env: CONDA_ENV=docs + - python: 3.6 + env: CONDA_ENV=py36-hypothesis allow_failures: - python: 3.6 env: @@ -104,6 +106,8 @@ script: - if [[ "$CONDA_ENV" == "docs" ]]; then conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc; sphinx-build -n -j auto -b html -d _build/doctrees doc _build/html; + elif [[ "$CONDA_ENV" == "py36-hypothesis" ]]; then + pytest properties ; else py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing --verbose $EXTRA_FLAGS; fi diff --git a/ci/requirements-py36-hypothesis.yml b/ci/requirements-py36-hypothesis.yml new file mode 100644 index 00000000000..29f4ae33538 --- /dev/null +++ b/ci/requirements-py36-hypothesis.yml @@ -0,0 +1,27 @@ +name: test_env +channels: + - conda-forge +dependencies: + - python=3.6 + - dask + - distributed + - h5py + - h5netcdf + - matplotlib + - netcdf4 + - pytest + - flake8 + - numpy + - pandas + - scipy + - seaborn + - toolz + - rasterio + - bottleneck + - zarr + - pip: + - coveralls + - pytest-cov + - pydap + - lxml + - hypothesis diff --git a/properties/README.md b/properties/README.md new file mode 100644 index 00000000000..711062a2473 --- /dev/null +++ b/properties/README.md @@ -0,0 +1,22 @@ +# Property-based tests using Hypothesis + +This directory contains property-based tests using a library +called [Hypothesis](https://github.com/HypothesisWorks/hypothesis-python). + +The property tests for Xarray are a work in progress - more are always welcome. +They are stored in a separate directory because they tend to run more examples +and thus take longer, and so that local development can run a test suite +without needing to `pip install hypothesis`. + +## Hang on, "property-based" tests? + +Instead of making assertions about operations on a particular piece of +data, you use Hypothesis to describe a *kind* of data, then make assertions +that should hold for *any* example of this kind. + +For example: "given a 2d ndarray of dtype uint8 `arr`, +`xr.DataArray(arr).plot.imshow()` never raises an exception". + +Hypothesis will then try many random examples, and report a minimised +failing input for each error it finds. +[See the docs for more info.](https://hypothesis.readthedocs.io/en/master/) diff --git a/properties/test_encode_decode.py b/properties/test_encode_decode.py new file mode 100644 index 00000000000..8d84c0f6815 --- /dev/null +++ b/properties/test_encode_decode.py @@ -0,0 +1,46 @@ +""" +Property-based tests for encoding/decoding methods. + +These ones pass, just as you'd hope! + +""" +from __future__ import absolute_import, division, print_function + +from hypothesis import given, settings +import hypothesis.strategies as st +import hypothesis.extra.numpy as npst + +import xarray as xr + +# Run for a while - arrays are a bigger search space than usual +settings.deadline = None + + +an_array = npst.arrays( + dtype=st.one_of( + npst.unsigned_integer_dtypes(), + npst.integer_dtypes(), + npst.floating_dtypes(), + ), + shape=npst.array_shapes(max_side=3), # max_side specified for performance +) + + +@given(st.data(), an_array) +def test_CFMask_coder_roundtrip(data, arr): + names = data.draw(st.lists(st.text(), min_size=arr.ndim, + max_size=arr.ndim, unique=True).map(tuple)) + original = xr.Variable(names, arr) + coder = xr.coding.variables.CFMaskCoder() + roundtripped = coder.decode(coder.encode(original)) + xr.testing.assert_identical(original, roundtripped) + + +@given(st.data(), an_array) +def test_CFScaleOffset_coder_roundtrip(data, arr): + names = data.draw(st.lists(st.text(), min_size=arr.ndim, + max_size=arr.ndim, unique=True).map(tuple)) + original = xr.Variable(names, arr) + coder = xr.coding.variables.CFScaleOffsetCoder() + roundtripped = coder.decode(coder.encode(original)) + xr.testing.assert_identical(original, roundtripped) From 85e240e8b9ff748493a279f7a3cc605bad001e26 Mon Sep 17 00:00:00 2001 From: Ed Doddridge Date: Wed, 21 Mar 2018 12:24:59 -0400 Subject: [PATCH 062/282] extraneous full stop in 88 char warning (#2003) --- xarray/backends/netCDF4_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 01d1a4de5f5..28aa4dbd121 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -265,7 +265,7 @@ def open(cls, filename, mode='r', format='NETCDF4', group=None, LooseVersion(nc4.__version__) < "1.3.1"): warnings.warn( '\nA segmentation fault may occur when the\n' - 'file path has exactly 88 characters as it does.\n' + 'file path has exactly 88 characters as it does\n' 'in this case. The issue is known to occur with\n' 'version 1.2.4 of netCDF4 and can be addressed by\n' 'upgrading netCDF4 to at least version 1.3.1.\n' From 9261601f89c0d3cfc54db16718c82399d95266bd Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 21 Mar 2018 21:36:55 -0700 Subject: [PATCH 063/282] Remove note about xray from docs index (#2000) We made this change several years ago now -- it's no longer timely news to share with users. --- doc/index.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index f4f036b8e58..7f7ed29179c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -17,12 +17,6 @@ pandas excels. Our approach adopts the `Common Data Model`_ for self- describing scientific data in widespread use in the Earth sciences: ``xarray.Dataset`` is an in-memory representation of a netCDF file. -.. note:: - - xray is now xarray! See :ref:`the v0.7.0 release notes` - for more details. The preferred URL for these docs is now - http://xarray.pydata.org. - .. _pandas: http://pandas.pydata.org .. _Common Data Model: http://www.unidata.ucar.edu/software/thredds/current/netcdf-java/CDM .. _netCDF: http://www.unidata.ucar.edu/software/netcdf From 7c2c43ce6b1792e1193635ab9b64fd248266f632 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 23 Mar 2018 15:51:56 -0700 Subject: [PATCH 064/282] Add weighted mean docs. (#2012) * Add weighted mean docs. Copied over from https://stackoverflow.com/questions/48510784/xarray-rolling-mean-with-weights * fix voice * fix --- doc/computation.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/computation.rst b/doc/computation.rst index 589df8eac36..0f22a2ed967 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -170,11 +170,11 @@ We can also manually iterate through ``Rolling`` objects: for label, arr_window in r: # arr_window is a view of x -Finally, the rolling object has ``construct`` method, which gives a -view of the original ``DataArray`` with the windowed dimension attached to +Finally, the rolling object has a ``construct`` method which returns a +view of the original ``DataArray`` with the windowed dimension in the last position. -You can use this for more advanced rolling operations, such as strided rolling, -windowed rolling, convolution, short-time FFT, etc. +You can use this for more advanced rolling operations such as strided rolling, +windowed rolling, convolution, short-time FFT etc. .. ipython:: python @@ -185,6 +185,12 @@ windowed rolling, convolution, short-time FFT, etc. Because the ``DataArray`` given by ``r.construct('window_dim')`` is a view of the original array, it is memory efficient. +You can also use ``construct`` to compute a weighted rolling mean: + +.. ipython:: python + + weight = xr.DataArray([0.25, 0.5, 0.25], dims=['window']) + arr.rolling(y=3).construct('window').dot(weight) .. note:: numpy's Nan-aggregation functions such as ``nansum`` copy the original array. From c4c683ff6bba9acc892c6ad73174550544276b09 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 28 Mar 2018 11:37:57 -0700 Subject: [PATCH 065/282] xfail tests that append to netCDF files with scipy (#2021) * xfail tests that append to netCDF files with scipy These are broken by SciPy 1.0.1. For details, see https://github.com/pydata/xarray/issues/2019 * xfail pynio tests that use scipy too --- xarray/tests/test_backends.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 5b9bb2a0506..0bef13a8c43 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1404,8 +1404,23 @@ def create_zarr_target(self): yield tmp +class ScipyWriteTest(CFEncodedDataTest, NetCDF3Only): + + def test_append_write(self): + import scipy + if scipy.__version__ == '1.0.1': + pytest.xfail('https://github.com/scipy/scipy/issues/8625') + super(ScipyWriteTest, self).test_append_write() + + def test_append_overwrite_values(self): + import scipy + if scipy.__version__ == '1.0.1': + pytest.xfail('https://github.com/scipy/scipy/issues/8625') + super(ScipyWriteTest, self).test_append_overwrite_values() + + @requires_scipy -class ScipyInMemoryDataTest(CFEncodedDataTest, NetCDF3Only, TestCase): +class ScipyInMemoryDataTest(ScipyWriteTest, TestCase): engine = 'scipy' @contextlib.contextmanager @@ -1431,7 +1446,7 @@ class ScipyInMemoryDataTestAutocloseTrue(ScipyInMemoryDataTest): @requires_scipy -class ScipyFileObjectTest(CFEncodedDataTest, NetCDF3Only, TestCase): +class ScipyFileObjectTest(ScipyWriteTest, TestCase): engine = 'scipy' @contextlib.contextmanager @@ -1459,7 +1474,7 @@ def test_pickle_dataarray(self): @requires_scipy -class ScipyFilePathTest(CFEncodedDataTest, NetCDF3Only, TestCase): +class ScipyFilePathTest(ScipyWriteTest, TestCase): engine = 'scipy' @contextlib.contextmanager @@ -2168,7 +2183,7 @@ def test_session(self): @requires_scipy @requires_pynio -class TestPyNio(CFEncodedDataTest, NetCDF3Only, TestCase): +class PyNioTest(ScipyWriteTest, TestCase): def test_write_store(self): # pynio is read-only for now pass @@ -2194,7 +2209,7 @@ def test_weakrefs(self): assert_identical(actual, expected) -class TestPyNioAutocloseTrue(TestPyNio): +class PyNioTestAutocloseTrue(PyNioTest): autoclose = True From 8e4231a28d8385e95c156f17ccfefeab537f63ed Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 28 Mar 2018 14:38:21 -0400 Subject: [PATCH 066/282] consolidate to one pytest config (#2020) don't even try collecting properties/ as this raises if hypothesis not installed --- pytest.ini | 2 -- setup.cfg | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 0132dbd4752..00000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = -p no:hypothesis diff --git a/setup.cfg b/setup.cfg index fe6e63a5080..ec30a10b242 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ universal = 1 [tool:pytest] python_files=test_*.py +testpaths=xarray/tests [flake8] max-line-length=79 From 44cc50d32b33016f453f2278b74e00a3c0852d5e Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 29 Mar 2018 20:44:54 -0700 Subject: [PATCH 067/282] Fix test_88_character_filename_segmentation_fault (#2026) * Fix test_88_character_filename_segmentation_fault This test turning all warnings into errors. Now it's more robust, and only converts the appropriate warning into an error. Fixes GH2025 * Fix bad git merge --- xarray/backends/netCDF4_.py | 14 +++++++------- xarray/tests/test_backends.py | 6 ++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 28aa4dbd121..89a0e72ef07 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -264,13 +264,13 @@ def open(cls, filename, mode='r', format='NETCDF4', group=None, if (len(filename) == 88 and LooseVersion(nc4.__version__) < "1.3.1"): warnings.warn( - '\nA segmentation fault may occur when the\n' - 'file path has exactly 88 characters as it does\n' - 'in this case. The issue is known to occur with\n' - 'version 1.2.4 of netCDF4 and can be addressed by\n' - 'upgrading netCDF4 to at least version 1.3.1.\n' - 'More details can be found here:\n' - 'https://github.com/pydata/xarray/issues/1745 \n') + 'A segmentation fault may occur when the ' + 'file path has exactly 88 characters as it does ' + 'in this case. The issue is known to occur with ' + 'version 1.2.4 of netCDF4 and can be addressed by ' + 'upgrading netCDF4 to at least version 1.3.1. ' + 'More details can be found here: ' + 'https://github.com/pydata/xarray/issues/1745') if format is None: format = 'NETCDF4' opener = functools.partial(_open_netcdf4_group, filename, mode=mode, diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 0bef13a8c43..e0f030368bb 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1137,8 +1137,10 @@ def test_88_character_filename_segmentation_fault(self): # should be fixed in netcdf4 v1.3.1 with mock.patch('netCDF4.__version__', '1.2.4'): with warnings.catch_warnings(): - warnings.simplefilter("error") - with raises_regex(Warning, 'segmentation fault'): + message = ('A segmentation fault may occur when the ' + 'file path has exactly 88 characters') + warnings.filterwarnings('error', message) + with pytest.raises(Warning): # Need to construct 88 character filepath xr.Dataset().to_netcdf('a' * (88 - len(os.getcwd()) - 1)) From 1b48ac87905676b18e947951b0cac23e70c7b40e Mon Sep 17 00:00:00 2001 From: Ryan May Date: Fri, 30 Mar 2018 19:15:59 -0600 Subject: [PATCH 068/282] Allow _FillValue and missing_value to differ (Fixes #1749) (#2016) * Allow _FillValue and missing_value to differ (Fixes #1749) The CF standard permits both values, and them to have different values, so we should not be treating this as an error--just mask out all of them. * Add whats-new entry --- doc/whats-new.rst | 4 ++++ xarray/coding/variables.py | 28 +++++++--------------------- xarray/tests/test_conventions.py | 9 ++++++--- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b28beb9e3b2..eb7d33c94e3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -40,6 +40,10 @@ Enhancements - Some speed improvement to construct :py:class:`~xarray.DataArrayRolling` object (:issue:`1993`) By `Keisuke Fujii `_. + - Handle variables with different values for ``missing_value`` and + ``_FillValue`` by masking values for both attributes; previously this + resulted in a ``ValueError``. (:issue:`2016`) + By `Ryan May `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/coding/variables.py b/xarray/coding/variables.py index ced535643a5..4e61e7d4722 100644 --- a/xarray/coding/variables.py +++ b/xarray/coding/variables.py @@ -7,7 +7,7 @@ import numpy as np import pandas as pd -from ..core import dtypes, duck_array_ops, indexing, utils +from ..core import dtypes, duck_array_ops, indexing from ..core.pycompat import dask_array_type from ..core.variable import Variable @@ -152,26 +152,12 @@ def encode(self, variable, name=None): def decode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_decoding(variable) - if 'missing_value' in attrs: - # missing_value is deprecated, but we still want to support it as - # an alias for _FillValue. - if ('_FillValue' in attrs and - not utils.equivalent(attrs['_FillValue'], - attrs['missing_value'])): - raise ValueError("Conflicting _FillValue and missing_value " - "attrs on a variable {!r}: {} vs. {}\n\n" - "Consider opening the offending dataset " - "using decode_cf=False, correcting the " - "attrs and decoding explicitly using " - "xarray.decode_cf()." - .format(name, attrs['_FillValue'], - attrs['missing_value'])) - attrs['_FillValue'] = attrs.pop('missing_value') - - if '_FillValue' in attrs: - raw_fill_value = pop_to(attrs, encoding, '_FillValue', name=name) - encoded_fill_values = [ - fv for fv in np.ravel(raw_fill_value) if not pd.isnull(fv)] + raw_fill_values = [pop_to(attrs, encoding, attr, name=name) + for attr in ('missing_value', '_FillValue')] + if raw_fill_values: + encoded_fill_values = {fv for option in raw_fill_values + for fv in np.ravel(option) + if not pd.isnull(fv)} if len(encoded_fill_values) > 1: warnings.warn("variable {!r} has multiple fill values {}, " diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 7028bac7057..5b82f5a26f2 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -156,12 +156,15 @@ def test(self): def test_decode_cf_with_conflicting_fill_missing_value(): - var = Variable(['t'], np.arange(10), + expected = Variable(['t'], [np.nan, np.nan, 2], {'units': 'foobar'}) + var = Variable(['t'], np.arange(3), {'units': 'foobar', 'missing_value': 0, '_FillValue': 1}) - with raises_regex(ValueError, "_FillValue and missing_value"): - conventions.decode_cf_variable('t', var) + with warnings.catch_warnings(record=True) as w: + actual = conventions.decode_cf_variable('t', var) + assert_identical(actual, expected) + assert 'has multiple fill' in str(w[0].message) expected = Variable(['t'], np.arange(10), {'units': 'foobar'}) From c78469a63417d9b7da0d5662788f10fb82923ea3 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Sat, 31 Mar 2018 03:16:14 +0200 Subject: [PATCH 069/282] Fix an overflow bug in decode_cf_datetime (#2015) * Fix an overflow bug in decode_cf_datetime * Better test * Other solution * Better test * Back to previous because of appveyor --- doc/whats-new.rst | 3 +++ xarray/coding/times.py | 3 ++- xarray/tests/test_coding_times.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index eb7d33c94e3..24ff240cea3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -51,6 +51,9 @@ Bug fixes - Fixed labeled indexing with slice bounds given by xarray objects with datetime64 or timedelta64 dtypes (:issue:`1240`). By `Stephan Hoyer `_. +- Fixed a bug in decode_cf_datetime where ``int32`` arrays weren't parsed + correctly (:issue:`2002`). + By `Fabien Maussion `_. .. _whats-new.0.10.2: diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 1bb4e31ae7e..8a1e9f82c6c 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -166,7 +166,8 @@ def decode_cf_datetime(num_dates, units, calendar=None): # Cast input dates to integers of nanoseconds because `pd.to_datetime` # works much faster when dealing with integers - flat_num_dates_ns_int = (flat_num_dates * + # make _NS_PER_TIME_DELTA an array to ensure type upcasting + flat_num_dates_ns_int = (flat_num_dates.astype(np.float64) * _NS_PER_TIME_DELTA[delta]).astype(np.int64) dates = (pd.to_timedelta(flat_num_dates_ns_int, 'ns') + diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index b85f92ece66..ab33329b51a 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -49,6 +49,7 @@ def test_cf_datetime(self): ([0.5, 1.5], 'hours since 1900-01-01T00:00:00'), (0, 'milliseconds since 2000-01-01T00:00:00'), (0, 'microseconds since 2000-01-01T00:00:00'), + (np.int32(788961600), 'seconds since 1981-01-01'), # GH2002 ]: for calendar in ['standard', 'gregorian', 'proleptic_gregorian']: expected = _ensure_naive_tz( From a90a7c5dd93a0e7b3d73bf8f272ae086acbff32d Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 2 Apr 2018 17:44:28 -0700 Subject: [PATCH 070/282] Raise an informative error message when converting Dataset -> np.ndarray (#2032) Makes `np.asarray(dataset)` issue an informative error. Currently, `np.asarray(xr.Dataset({'x': 0}))` raises `KeyError: 0`, which makes no sense. --- doc/whats-new.rst | 3 +++ xarray/core/dataset.py | 6 ++++++ xarray/tests/test_dataset.py | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 24ff240cea3..c22b252728f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -51,6 +51,9 @@ Bug fixes - Fixed labeled indexing with slice bounds given by xarray objects with datetime64 or timedelta64 dtypes (:issue:`1240`). By `Stephan Hoyer `_. +- Attempting to convert an xarray.Dataset into a numpy array now raises an + informative error message. + By `Stephan Hoyer `_. - Fixed a bug in decode_cf_datetime where ``int32`` arrays weren't parsed correctly (:issue:`2002`). By `Fabien Maussion `_. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index e960d433f98..f28e7980b34 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -849,6 +849,12 @@ def __iter__(self): FutureWarning, stacklevel=2) return iter(self._variables) + def __array__(self, dtype=None): + raise TypeError('cannot directly convert an xarray.Dataset into a ' + 'numpy array. Instead, create an xarray.DataArray ' + 'first, either with indexing on the Dataset or by ' + 'invoking the `to_array()` method.') + @property def nbytes(self): return sum(v.nbytes for v in self.variables.values()) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index b4ca14d2384..826f50003fa 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -443,6 +443,11 @@ def test_properties(self): assert Dataset({'x': np.int64(1), 'y': np.float32([1, 2])}).nbytes == 16 + def test_asarray(self): + ds = Dataset({'x': 0}) + with raises_regex(TypeError, 'cannot directly convert'): + np.asarray(ds) + def test_get_index(self): ds = Dataset({'foo': (('x', 'y'), np.zeros((2, 3)))}, coords={'x': ['a', 'b']}) From 8c194b695f07adad43e7a3aa5c59f53b01131807 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 2 Apr 2018 20:59:04 -0700 Subject: [PATCH 071/282] Include xarray logo on all docs pages (#2033) * Include xarray logo on all docs pages * Set html_theme even when on rtd * Add missing theme files --- doc/_static/style.css | 18 ++++++++++++++++++ doc/_templates/layout.html | 2 ++ doc/conf.py | 18 +++++------------- doc/index.rst | 10 ++-------- 4 files changed, 27 insertions(+), 21 deletions(-) create mode 100644 doc/_static/style.css create mode 100644 doc/_templates/layout.html diff --git a/doc/_static/style.css b/doc/_static/style.css new file mode 100644 index 00000000000..7257d57db66 --- /dev/null +++ b/doc/_static/style.css @@ -0,0 +1,18 @@ +@import url("theme.css"); + +.wy-side-nav-search>a img.logo, +.wy-side-nav-search .wy-dropdown>a img.logo { + width: 12rem +} + +.wy-side-nav-search { + background-color: #eee; +} + +.wy-side-nav-search>div.version { + display: none; +} + +.wy-nav-top { + background-color: #555; +} diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html new file mode 100644 index 00000000000..4c57ba83056 --- /dev/null +++ b/doc/_templates/layout.html @@ -0,0 +1,2 @@ +{% extends "!layout.html" %} +{% set css_files = css_files + ["_static/style.css"] %} diff --git a/doc/conf.py b/doc/conf.py index 2f6849fd0bd..0fd5eaf05d7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -149,22 +149,14 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. - -# on_rtd is whether we are on readthedocs.org, this line of code grabbed from -# docs.readthedocs.org -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - -# otherwise, readthedocs.org uses their theme by default, so no need to specify it +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + 'logo_only': True, +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] @@ -178,7 +170,7 @@ # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +html_logo = "_static/dataset-diagram-logo.png" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 diff --git a/doc/index.rst b/doc/index.rst index 7f7ed29179c..dc00c548b35 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,11 +1,5 @@ -.. image:: _static/dataset-diagram-logo.png - :width: 300 px - :align: center - -| - -N-D labeled arrays and datasets in Python -========================================= +xarray: N-D labeled arrays and datasets in Python +================================================= **xarray** (formerly **xray**) is an open source project and Python package that aims to bring the labeled data power of pandas_ to the physical sciences, From a5f7d6ac60e8e5682e2be739dd520b7a3bbd0fc7 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Tue, 3 Apr 2018 22:46:56 -0400 Subject: [PATCH 072/282] Isin (#2031) * gitignore testmon * initial isin implementation * gitignore * dask * numpy version check not needed * numpy version check for isin * move to common * rename data_set to ds * Revert "rename data_set to ds" This reverts commit 75493c2e9e1eb9b39a2222a19a7725afe1f009be. * 'expect' test for dataset * unneeded import * formatting * docs * Raise an informative error message when converting Dataset -> np.ndarray Makes `np.asarray(dataset)` issue an informative error. Currently, `np.asarray(xr.Dataset({'x': 0}))` raises `KeyError: 0`, which makes no sense. * normal tests are better than a weird middle ground * dask test * grammar * try changing skip decorator ordering * just use has_dask * another noqa? * flake for py3.4 * flake --- .gitignore | 5 +-- doc/api.rst | 2 ++ doc/whats-new.rst | 7 +++- xarray/core/common.py | 30 ++++++++++++++++ xarray/tests/test_dataarray.py | 33 +++++++++++++++++ xarray/tests/test_dataset.py | 65 +++++++++++++++++++++++++++++++--- xarray/tests/test_variable.py | 2 -- 7 files changed, 135 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 9069cbebacc..70458f00648 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,7 @@ nosetests.xml .cache .ropeproject/ .tags* -.testmondata +.testmon* .pytest_cache # asv environments @@ -51,10 +51,11 @@ nosetests.xml .project .pydevproject -# PyCharm and Vim +# IDEs .idea *.swp .DS_Store +.vscode/ # xarray specific doc/_build diff --git a/doc/api.rst b/doc/api.rst index 8ee5d548892..9772ed6ba62 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -174,6 +174,7 @@ Computation :py:attr:`~Dataset.cumsum` :py:attr:`~Dataset.cumprod` :py:attr:`~Dataset.rank` +:py:attr:`~Dataset.isin` **Grouped operations**: :py:attr:`~core.groupby.DatasetGroupBy.assign` @@ -339,6 +340,7 @@ Computation :py:attr:`~DataArray.cumsum` :py:attr:`~DataArray.cumprod` :py:attr:`~DataArray.rank` +:py:attr:`~DataArray.isin` **Grouped operations**: :py:attr:`~core.groupby.DataArrayGroupBy.assign_coords` diff --git a/doc/whats-new.rst b/doc/whats-new.rst index c22b252728f..3f01f48bb0f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -37,7 +37,12 @@ Documentation Enhancements ~~~~~~~~~~~~ - - Some speed improvement to construct :py:class:`~xarray.DataArrayRolling` +- `~xarray.DataArray.isin` and `~xarray.Dataset.isin` methods, which test each value + in the array for whether it is contained in the supplied list, returning a bool array. + Similar to the ``np.isin`` function. Requires NumPy >= 1.13 +By `Maximilian Roos ` + +- Some speed improvement to construct :py:class:`~xarray.DataArrayRolling` object (:issue:`1993`) By `Keisuke Fujii `_. - Handle variables with different values for ``missing_value`` and diff --git a/xarray/core/common.py b/xarray/core/common.py index 337c1c51415..5904dcb1274 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function import warnings +from distutils.version import LooseVersion import numpy as np import pandas as pd @@ -744,6 +745,35 @@ def close(self): self._file_obj.close() self._file_obj = None + def isin(self, test_elements): + """Tests each value in the array for whether it is in the supplied list + Requires NumPy >= 1.13 + + Parameters + ---------- + element : array_like + Input array. + test_elements : array_like + The values against which to test each value of `element`. + This argument is flattened if an array or array_like. + See numpy notes for behavior with non-array-like parameters. + + ------- + isin : same as object, bool + Has the same shape as object + """ + if LooseVersion(np.__version__) < LooseVersion('1.13.0'): + raise ImportError('isin requires numpy version 1.13.0 or later') + from .computation import apply_ufunc + + return apply_ufunc( + np.isin, + self, + kwargs=dict(test_elements=test_elements), + dask='parallelized', + output_dtypes=[np.bool_], + ) + def __enter__(self): return self diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 3fd229cf394..b065efb084c 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3327,6 +3327,14 @@ def da(request): [0, np.nan, 1, 2, np.nan, 3, 4, 5, np.nan, 6, 7], dims='time') + if request.param == 'repeating_ints': + return DataArray( + np.tile(np.arange(12), 5).reshape(5, 4, 3), + coords={'x': list('abc'), + 'y': list('defg')}, + dims=list('zyx') + ) + @pytest.fixture def da_dask(seed=123): @@ -3339,6 +3347,31 @@ def da_dask(seed=123): return da +@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13.0'), + reason='requires numpy version 1.13.0 or later') +@pytest.mark.parametrize('da', ('repeating_ints', ), indirect=True) +def test_isin(da): + + expected = DataArray( + np.asarray([[0, 0, 0], [1, 0, 0]]), + dims=list('yx'), + coords={'x': list('abc'), + 'y': list('de')}, + ).astype('bool') + + result = da.isin([3]).sel(y=list('de'), z=0) + assert_equal(result, expected) + + expected = DataArray( + np.asarray([[0, 0, 1], [1, 0, 0]]), + dims=list('yx'), + coords={'x': list('abc'), + 'y': list('de')}, + ).astype('bool') + result = da.isin([2, 3]).sel(y=list('de'), z=0) + assert_equal(result, expected) + + @pytest.mark.parametrize('da', (1, 2), indirect=True) def test_rolling_iter(da): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 826f50003fa..a7b55735579 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -21,7 +21,7 @@ from . import ( InaccessibleArray, TestCase, UnexpectedDataAccess, assert_allclose, - assert_array_equal, assert_equal, assert_identical, raises_regex, + assert_array_equal, assert_equal, assert_identical, has_dask, raises_regex, requires_bottleneck, requires_dask, requires_scipy, source_ndarray) try: @@ -4037,9 +4037,66 @@ def test_ipython_key_completion(self): # Py.test tests -@pytest.fixture() -def data_set(seed=None): - return create_test_data(seed) +@pytest.fixture(params=[None]) +def data_set(request): + return create_test_data(request.param) + + +@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13.0'), + reason='requires numpy version 1.13.0 or later') +@pytest.mark.parametrize('test_elements', ( + [1, 2], + np.array([1, 2]), + DataArray([1, 2]), + pytest.mark.xfail(Dataset({'x': [1, 2]})), +)) +def test_isin(test_elements): + expected = Dataset( + data_vars={ + 'var1': (('dim1',), [0, 1]), + 'var2': (('dim1',), [1, 1]), + 'var3': (('dim1',), [0, 1]), + } + ).astype('bool') + + result = Dataset( + data_vars={ + 'var1': (('dim1',), [0, 1]), + 'var2': (('dim1',), [1, 2]), + 'var3': (('dim1',), [0, 1]), + } + ).isin(test_elements) + + assert_equal(result, expected) + + +@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13.0') or # noqa + not has_dask, # noqa + reason='requires dask and numpy version 1.13.0 or later') +@pytest.mark.parametrize('test_elements', ( + [1, 2], + np.array([1, 2]), + DataArray([1, 2]), + pytest.mark.xfail(Dataset({'x': [1, 2]})), +)) +def test_isin_dask(test_elements): + expected = Dataset( + data_vars={ + 'var1': (('dim1',), [0, 1]), + 'var2': (('dim1',), [1, 1]), + 'var3': (('dim1',), [0, 1]), + } + ).astype('bool') + + result = Dataset( + data_vars={ + 'var1': (('dim1',), [0, 1]), + 'var2': (('dim1',), [1, 2]), + 'var3': (('dim1',), [0, 1]), + } + ).chunk(1).isin(test_elements).compute() + + assert_equal(result, expected) def test_dir_expected_attrs(data_set): diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index c4489f50246..722d1af14f7 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1422,8 +1422,6 @@ def test_reduce(self): with raises_regex(ValueError, 'cannot supply both'): v.mean(dim='x', axis=0) - @pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.10.0'), - reason='requires numpy version 1.10.0 or later') def test_quantile(self): v = Variable(['x', 'y'], self.d) for q in [0.25, [0.50], [0.25, 0.75]]: From 6402391cf206fd04c12d44773fecd9b42ea0c246 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 6 Apr 2018 08:36:48 -0700 Subject: [PATCH 073/282] isin: better docs, support older numpy and use dask.array.isin. (#2038) * isin: better docs, support older numpy and use dask.array.isin. I added a section to the indexing docs showing how to use `isin` with `where` to do idnexing. Adding support for older numpy turned out to be pretty easy, so I added a copy of np.isin in npcompat. This PR also makes use of dask.array.isin, so that we can support a dask array properly as the second argument: https://github.com/dask/dask/pull/3363 * lint * more lint * yet more lint --- doc/api.rst | 4 +- doc/indexing.rst | 25 +++++++++ doc/whats-new.rst | 22 ++++---- licenses/DASK_LICENSE | 28 ++++++++++ xarray/core/common.py | 42 ++++++++++---- xarray/core/dask_array_compat.py | 34 +++++++++++ xarray/core/duck_array_ops.py | 24 ++++---- xarray/core/npcompat.py | 96 ++++++++++++++++++++++++++++++++ xarray/tests/test_dataarray.py | 2 - xarray/tests/test_dataset.py | 14 ++--- 10 files changed, 248 insertions(+), 43 deletions(-) create mode 100644 licenses/DASK_LICENSE create mode 100644 xarray/core/dask_array_compat.py diff --git a/doc/api.rst b/doc/api.rst index 9772ed6ba62..bce4e0d1c8e 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -132,6 +132,7 @@ Missing value handling Dataset.bfill Dataset.interpolate_na Dataset.where + Dataset.isin Computation ----------- @@ -174,7 +175,6 @@ Computation :py:attr:`~Dataset.cumsum` :py:attr:`~Dataset.cumprod` :py:attr:`~Dataset.rank` -:py:attr:`~Dataset.isin` **Grouped operations**: :py:attr:`~core.groupby.DatasetGroupBy.assign` @@ -285,6 +285,7 @@ Missing value handling DataArray.bfill DataArray.interpolate_na DataArray.where + DataArray.isin Comparisons ----------- @@ -340,7 +341,6 @@ Computation :py:attr:`~DataArray.cumsum` :py:attr:`~DataArray.cumprod` :py:attr:`~DataArray.rank` -:py:attr:`~DataArray.isin` **Grouped operations**: :py:attr:`~core.groupby.DataArrayGroupBy.assign_coords` diff --git a/doc/indexing.rst b/doc/indexing.rst index 6b01471ecfb..6bdffde23c2 100644 --- a/doc/indexing.rst +++ b/doc/indexing.rst @@ -265,6 +265,31 @@ elements that are fully masked: arr2.where(arr2.y < 2, drop=True) +.. _selecting values with isin: + +Selecting values with ``isin`` +------------------------------ + +To check whether elements of an xarray object contain a single object, you can +compare with the equality operator ``==`` (e.g., ``arr == 3``). To check +multiple values, use :py:meth:`~xarray.DataArray.isin`: + +.. ipython:: python + + arr = xr.DataArray([1, 2, 3, 4, 5], dims=['x']) + arr.isin([2, 4]) + +:py:meth:`~xarray.DataArray.isin` works particularly well with +:py:meth:`~xarray.DataArray.where` to support indexing by arrays that are not +already labels of an array: + +.. ipython:: python + + lookup = xr.DataArray([-1, -2, -3, -4, -5], dims=['x']) + arr.where(lookup.isin([-2, -4]), drop=True) + +However, some caution is in order: when done repeatedly, this type of indexing +is significantly slower than using :py:meth:`~xarray.DataArray.sel`. .. _vectorized_indexing: diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3f01f48bb0f..b3130a1a4ab 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -37,18 +37,20 @@ Documentation Enhancements ~~~~~~~~~~~~ -- `~xarray.DataArray.isin` and `~xarray.Dataset.isin` methods, which test each value - in the array for whether it is contained in the supplied list, returning a bool array. - Similar to the ``np.isin`` function. Requires NumPy >= 1.13 -By `Maximilian Roos ` +- :py:meth:`~xarray.DataArray.isin` and :py:meth:`~xarray.Dataset.isin` methods, + which test each value in the array for whether it is contained in the + supplied list, returning a bool array. See :ref:`selecting values with isin` + for full details. Similar to the ``np.isin`` function. + By `Maximilian Roos `_. - Some speed improvement to construct :py:class:`~xarray.DataArrayRolling` - object (:issue:`1993`) - By `Keisuke Fujii `_. - - Handle variables with different values for ``missing_value`` and - ``_FillValue`` by masking values for both attributes; previously this - resulted in a ``ValueError``. (:issue:`2016`) - By `Ryan May `_. + object (:issue:`1993`) + By `Keisuke Fujii `_. + +- Handle variables with different values for ``missing_value`` and + ``_FillValue`` by masking values for both attributes; previously this + resulted in a ``ValueError``. (:issue:`2016`) + By `Ryan May `_. Bug fixes ~~~~~~~~~ diff --git a/licenses/DASK_LICENSE b/licenses/DASK_LICENSE new file mode 100644 index 00000000000..893bddfb933 --- /dev/null +++ b/licenses/DASK_LICENSE @@ -0,0 +1,28 @@ +:py:meth:`~xarray.DataArray.isin`Copyright (c) 2014-2018, Anaconda, Inc. and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of Anaconda nor the names of any contributors may be used to +endorse or promote products derived from this software without specific prior +written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/xarray/core/common.py b/xarray/core/common.py index 5904dcb1274..5beb5234d4c 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from . import dtypes, formatting, ops +from . import duck_array_ops, dtypes, formatting, ops from .arithmetic import SupportsArithmetic from .pycompat import OrderedDict, basestring, dask_array_type, suppress from .utils import Frozen, SortedKeysDict @@ -746,32 +746,52 @@ def close(self): self._file_obj = None def isin(self, test_elements): - """Tests each value in the array for whether it is in the supplied list - Requires NumPy >= 1.13 + """Tests each value in the array for whether it is in the supplied list. Parameters ---------- - element : array_like - Input array. test_elements : array_like The values against which to test each value of `element`. This argument is flattened if an array or array_like. See numpy notes for behavior with non-array-like parameters. + Returns ------- isin : same as object, bool - Has the same shape as object + Has the same shape as this object. + + Examples + -------- + + >>> array = xr.DataArray([1, 2, 3], dims='x') + >>> array.isin([1, 3]) + + array([ True, False, True]) + Dimensions without coordinates: x + + See also + -------- + numpy.isin """ - if LooseVersion(np.__version__) < LooseVersion('1.13.0'): - raise ImportError('isin requires numpy version 1.13.0 or later') from .computation import apply_ufunc + from .dataset import Dataset + from .dataarray import DataArray + from .variable import Variable + + if isinstance(test_elements, Dataset): + raise TypeError( + 'isin() argument must be convertible to an array: {}' + .format(test_elements)) + elif isinstance(test_elements, (Variable, DataArray)): + # need to explicitly pull out data to support dask arrays as the + # second argument + test_elements = test_elements.data return apply_ufunc( - np.isin, + duck_array_ops.isin, self, kwargs=dict(test_elements=test_elements), - dask='parallelized', - output_dtypes=[np.bool_], + dask='allowed', ) def __enter__(self): diff --git a/xarray/core/dask_array_compat.py b/xarray/core/dask_array_compat.py new file mode 100644 index 00000000000..f6cb488aa4a --- /dev/null +++ b/xarray/core/dask_array_compat.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import, division, print_function + +from functools import wraps +import numpy as np +import dask.array as da + +try: + from dask.array import isin +except ImportError: # pragma: no cover + # Copied from dask v0.17.3. + # Used under the terms of Dask's license, see licenses/DASK_LICENSE. + + def _isin_kernel(element, test_elements, assume_unique=False): + values = np.in1d(element.ravel(), test_elements, + assume_unique=assume_unique) + return values.reshape(element.shape + (1,) * test_elements.ndim) + + @wraps(getattr(np, 'isin', None)) + def isin(element, test_elements, assume_unique=False, invert=False): + element = da.asarray(element) + test_elements = da.asarray(test_elements) + element_axes = tuple(range(element.ndim)) + test_axes = tuple(i + element.ndim for i in range(test_elements.ndim)) + mapped = da.atop(_isin_kernel, element_axes + test_axes, + element, element_axes, + test_elements, test_axes, + adjust_chunks={axis: lambda _: 1 + for axis in test_axes}, + dtype=bool, + assume_unique=assume_unique) + result = mapped.any(axis=test_axes) + if invert: + result = ~result + return result diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 3a5c4a124d1..faea30cdd99 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -26,23 +26,24 @@ has_bottleneck = False try: - import dask.array as da - has_dask = True + import dask.array as dask_array + from . import dask_array_compat except ImportError: - has_dask = False + dask_array = None + dask_array_compat = None -def _dask_or_eager_func(name, eager_module=np, list_of_args=False, - n_array_args=1): +def _dask_or_eager_func(name, eager_module=np, dask_module=dask_array, + list_of_args=False, n_array_args=1): """Create a function that dispatches to dask for dask array inputs.""" - if has_dask: + if dask_module is not None: def f(*args, **kwargs): if list_of_args: dispatch_args = args[0] else: dispatch_args = args[:n_array_args] - if any(isinstance(a, da.Array) for a in dispatch_args): - module = da + if any(isinstance(a, dask_array.Array) for a in dispatch_args): + module = dask_module else: module = eager_module return getattr(module, name)(*args, **kwargs) @@ -63,8 +64,8 @@ def fail_on_dask_array_input(values, msg=None, func_name=None): around = _dask_or_eager_func('around') isclose = _dask_or_eager_func('isclose') -notnull = _dask_or_eager_func('notnull', pd) -_isnull = _dask_or_eager_func('isnull', pd) +notnull = _dask_or_eager_func('notnull', eager_module=pd) +_isnull = _dask_or_eager_func('isnull', eager_module=pd) def isnull(data): @@ -80,7 +81,8 @@ def isnull(data): transpose = _dask_or_eager_func('transpose') _where = _dask_or_eager_func('where', n_array_args=3) -insert = _dask_or_eager_func('insert') +isin = _dask_or_eager_func('isin', eager_module=npcompat, + dask_module=dask_array_compat, n_array_args=2) take = _dask_or_eager_func('take') broadcast_to = _dask_or_eager_func('broadcast_to') diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index af722924aae..ec8adfffbf8 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -255,3 +255,99 @@ def flip(m, axis): raise ValueError("axis=%i is invalid for the %i-dimensional " "input array" % (axis, m.ndim)) return m[tuple(indexer)] + +try: + from numpy import isin +except ImportError: + + def isin(element, test_elements, assume_unique=False, invert=False): + """ + Calculates `element in test_elements`, broadcasting over `element` + only. Returns a boolean array of the same shape as `element` that is + True where an element of `element` is in `test_elements` and False + otherwise. + + Parameters + ---------- + element : array_like + Input array. + test_elements : array_like + The values against which to test each value of `element`. + This argument is flattened if it is an array or array_like. + See notes for behavior with non-array-like parameters. + assume_unique : bool, optional + If True, the input arrays are both assumed to be unique, which + can speed up the calculation. Default is False. + invert : bool, optional + If True, the values in the returned array are inverted, as if + calculating `element not in test_elements`. Default is False. + ``np.isin(a, b, invert=True)`` is equivalent to (but faster + than) ``np.invert(np.isin(a, b))``. + + Returns + ------- + isin : ndarray, bool + Has the same shape as `element`. The values `element[isin]` + are in `test_elements`. + + See Also + -------- + in1d : Flattened version of this function. + numpy.lib.arraysetops : Module with a number of other functions for + performing set operations on arrays. + + Notes + ----- + + `isin` is an element-wise function version of the python keyword `in`. + ``isin(a, b)`` is roughly equivalent to + ``np.array([item in b for item in a])`` if `a` and `b` are 1-D + sequences. + + `element` and `test_elements` are converted to arrays if they are not + already. If `test_elements` is a set (or other non-sequence collection) + it will be converted to an object array with one element, rather than + an array of the values contained in `test_elements`. This is a + consequence of the `array` constructor's way of handling non-sequence + collections. Converting the set to a list usually gives the desired + behavior. + + .. versionadded:: 1.13.0 + + Examples + -------- + >>> element = 2*np.arange(4).reshape((2, 2)) + >>> element + array([[0, 2], + [4, 6]]) + >>> test_elements = [1, 2, 4, 8] + >>> mask = np.isin(element, test_elements) + >>> mask + array([[ False, True], + [ True, False]]) + >>> element[mask] + array([2, 4]) + >>> mask = np.isin(element, test_elements, invert=True) + >>> mask + array([[ True, False], + [ False, True]]) + >>> element[mask] + array([0, 6]) + + Because of how `array` handles sets, the following does not + work as expected: + + >>> test_set = {1, 2, 4, 8} + >>> np.isin(element, test_set) + array([[ False, False], + [ False, False]]) + + Casting the set to a list gives the expected result: + + >>> np.isin(element, list(test_set)) + array([[ False, True], + [ True, False]]) + """ + element = np.asarray(element) + return np.in1d(element, test_elements, assume_unique=assume_unique, + invert=invert).reshape(element.shape) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index b065efb084c..32ab3a634cb 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3347,8 +3347,6 @@ def da_dask(seed=123): return da -@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13.0'), - reason='requires numpy version 1.13.0 or later') @pytest.mark.parametrize('da', ('repeating_ints', ), indirect=True) def test_isin(da): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index a7b55735579..cbd1cd33f2c 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -4042,13 +4042,10 @@ def data_set(request): return create_test_data(request.param) -@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13.0'), - reason='requires numpy version 1.13.0 or later') @pytest.mark.parametrize('test_elements', ( [1, 2], np.array([1, 2]), DataArray([1, 2]), - pytest.mark.xfail(Dataset({'x': [1, 2]})), )) def test_isin(test_elements): expected = Dataset( @@ -4070,14 +4067,11 @@ def test_isin(test_elements): assert_equal(result, expected) -@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13.0') or # noqa - not has_dask, # noqa - reason='requires dask and numpy version 1.13.0 or later') +@pytest.mark.skipif(not has_dask, reason='requires dask') @pytest.mark.parametrize('test_elements', ( [1, 2], np.array([1, 2]), DataArray([1, 2]), - pytest.mark.xfail(Dataset({'x': [1, 2]})), )) def test_isin_dask(test_elements): expected = Dataset( @@ -4099,6 +4093,12 @@ def test_isin_dask(test_elements): assert_equal(result, expected) +def test_isin_dataset(): + ds = Dataset({'x': [1, 2]}) + with pytest.raises(TypeError): + ds.isin(ds) + + def test_dir_expected_attrs(data_set): some_expected_attrs = {'pipe', 'mean', 'isnull', 'var1', From 72b4e211712e92e139db8b5ba0e2c7a7354deae7 Mon Sep 17 00:00:00 2001 From: Benjamin Root Date: Tue, 10 Apr 2018 16:48:58 -0400 Subject: [PATCH 074/282] concat_dim for auto_combine for a single object is now respected (#2048) * concat_dim for auto_combine for a single object is now respected Closes #1988 * Added what's new entry for the bugfix. --- doc/whats-new.rst | 4 ++++ xarray/core/combine.py | 3 ++- xarray/tests/test_backends.py | 14 ++++++++++++++ xarray/tests/test_combine.py | 19 +++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b3130a1a4ab..fcfe4887485 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -64,6 +64,10 @@ Bug fixes - Fixed a bug in decode_cf_datetime where ``int32`` arrays weren't parsed correctly (:issue:`2002`). By `Fabien Maussion `_. +- When calling `xr.auto_combine()` or `xr.open_mfdataset()` with a `concat_dim`, + the resulting dataset will have that one-element dimension (it was + silently dropped, previously) (:issue:`1988`). + By `Ben Root `_. .. _whats-new.0.10.2: diff --git a/xarray/core/combine.py b/xarray/core/combine.py index 8c1c58e9a40..430f0e564d6 100644 --- a/xarray/core/combine.py +++ b/xarray/core/combine.py @@ -340,7 +340,8 @@ def _dataarray_concat(arrays, dim, data_vars, coords, compat, def _auto_concat(datasets, dim=None, data_vars='all', coords='different'): - if len(datasets) == 1: + if len(datasets) == 1 and dim is None: + # There is nothing more to combine, so kick out early. return datasets[0] else: if dim is None: diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index e0f030368bb..a58104deef5 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2039,6 +2039,20 @@ def test_open_dataset(self): self.assertIsInstance(actual.foo.variable.data, np.ndarray) assert_identical(original, actual) + def test_open_single_dataset(self): + # Test for issue GH #1988. This makes sure that the + # concat_dim is utilized when specified in open_mfdataset(). + rnddata = np.random.randn(10) + original = Dataset({'foo': ('x', rnddata)}) + dim = DataArray([100], name='baz', dims='baz') + expected = Dataset({'foo': (('baz', 'x'), rnddata[np.newaxis, :])}, + {'baz': [100]}) + with create_tmp_file() as tmp: + original.to_netcdf(tmp) + with open_mfdataset([tmp], concat_dim=dim, + autoclose=self.autoclose) as actual: + assert_identical(expected, actual) + def test_dask_roundtrip(self): with create_tmp_file() as tmp: data = create_test_data() diff --git a/xarray/tests/test_combine.py b/xarray/tests/test_combine.py index 09918d9a065..482a280b355 100644 --- a/xarray/tests/test_combine.py +++ b/xarray/tests/test_combine.py @@ -377,3 +377,22 @@ def test_auto_combine_no_concat(self): data = Dataset({'x': 0}) actual = auto_combine([data, data, data], concat_dim=None) assert_identical(data, actual) + + # Single object, with a concat_dim explicitly provided + # Test the issue reported in GH #1988 + objs = [Dataset({'x': 0, 'y': 1})] + dim = DataArray([100], name='baz', dims='baz') + actual = auto_combine(objs, concat_dim=dim) + expected = Dataset({'x': ('baz', [0]), 'y': ('baz', [1])}, + {'baz': [100]}) + assert_identical(expected, actual) + + # Just making sure that auto_combine is doing what is + # expected for non-scalar values, too. + objs = [Dataset({'x': ('z', [0, 1]), 'y': ('z', [1, 2])})] + dim = DataArray([100], name='baz', dims='baz') + actual = auto_combine(objs, concat_dim=dim) + expected = Dataset({'x': (('baz', 'z'), [[0, 1]]), + 'y': (('baz', 'z'), [[1, 2]])}, + {'baz': [100]}) + assert_identical(expected, actual) From 9b76f219ec314dcb0c9a310c097a34f5c751fdd6 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 10 Apr 2018 20:56:49 -0700 Subject: [PATCH 075/282] xfail test_cross_engine_read_write_netcdf3 (#2051) This should fix the test suite on Travis-CI again. --- xarray/tests/test_backends.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a58104deef5..c5e8d126e43 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1578,6 +1578,7 @@ def test_engine(self): with raises_regex(ValueError, 'can only read'): open_dataset(BytesIO(netcdf_bytes), engine='foobar') + @pytest.mark.xfail(reason='https://github.com/pydata/xarray/issues/2050') def test_cross_engine_read_write_netcdf3(self): data = create_test_data() valid_engines = set() From e63e0e9766c68d7eef215da6f6d7b7dbaca4b652 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 11 Apr 2018 17:37:46 -0400 Subject: [PATCH 076/282] character out of place preventing formatting (#2052) --- xarray/core/computation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 685a3c66c54..7e9e7273229 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -698,7 +698,7 @@ def apply_ufunc(func, *args, **kwargs): on each input argument that should not be broadcast. By default, we assume there are no core dimensions on any input arguments. - For example ,``input_core_dims=[[], ['time']]`` indicates that all + For example, ``input_core_dims=[[], ['time']]`` indicates that all dimensions on the first argument and all dimensions other than 'time' on the second argument should be broadcast. From 8f96db41c177933b184e78d66f7148f1d0947cd9 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 11 Apr 2018 14:38:50 -0700 Subject: [PATCH 077/282] Add reference on MCVE to GitHub issue template (#2041) --- .github/ISSUE_TEMPLATE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d10e857c4ed..c7236b8159a 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,5 +1,8 @@ #### Code Sample, a copy-pastable example if possible +A "Minimal, Complete and Verifiable Example" will make it much easier for maintainers to help you: +http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports + ```python # Your code here From a9d1f3a36229636f0d519eb36a8d4a7c91f6e1cd Mon Sep 17 00:00:00 2001 From: Ryan Abernathey Date: Thu, 12 Apr 2018 19:38:01 -0400 Subject: [PATCH 078/282] Fix decode cf with dask (#2047) * fixes #1372 * what's new --- doc/whats-new.rst | 2 ++ xarray/conventions.py | 7 ++++--- xarray/tests/test_conventions.py | 13 ++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fcfe4887485..efe8277e922 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -55,6 +55,8 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed ``decode_cf`` function to operate lazily on dask arrays + (:issue:`1372`). By `Ryan Abernathey `_. - Fixed labeled indexing with slice bounds given by xarray objects with datetime64 or timedelta64 dtypes (:issue:`1240`). By `Stephan Hoyer `_. diff --git a/xarray/conventions.py b/xarray/conventions.py index 93fd56ed5d2..04664435732 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -9,7 +9,7 @@ from .coding import times, variables from .coding.variables import SerializationWarning from .core import duck_array_ops, indexing -from .core.pycompat import OrderedDict, basestring, iteritems +from .core.pycompat import OrderedDict, basestring, iteritems, dask_array_type from .core.variable import IndexVariable, Variable, as_variable @@ -490,8 +490,9 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True, del attributes['dtype'] data = BoolTypeArray(data) - return Variable(dimensions, indexing.LazilyOuterIndexedArray(data), - attributes, encoding=encoding) + if not isinstance(data, dask_array_type): + data = indexing.LazilyOuterIndexedArray(data) + return Variable(dimensions, data, attributes, encoding=encoding) def decode_cf_variables(variables, attributes, concat_characters=True, diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 5b82f5a26f2..86240f4e33e 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -18,7 +18,7 @@ from . import ( IndexerMaker, TestCase, assert_array_equal, raises_regex, requires_netCDF4, - requires_netcdftime, unittest) + requires_netcdftime, unittest, requires_dask) from .test_backends import CFEncodedDataTest B = IndexerMaker(indexing.BasicIndexer) @@ -332,6 +332,17 @@ def test_decode_cf_datetime_transition_to_invalid(self): assert_array_equal(ds_decoded.time.values, expected) + @requires_dask + def test_decode_cf_with_dask(self): + import dask + original = Dataset({ + 't': ('t', [0, 1, 2], {'units': 'days since 2000-01-01'}), + 'foo': ('t', [0, 0, 0], {'coordinates': 'y', 'units': 'bar'}), + 'y': ('t', [5, 10, -999], {'_FillValue': -999}) + }).chunk({'t': 1}) + decoded = conventions.decode_cf(original) + assert dask.is_dask_collection(decoded.y.data) + class CFEncodedInMemoryStore(WritableCFDataStore, InMemoryDataStore): pass From 295dd20028b1607ad810eae947a20d8122d311c2 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 13 Apr 2018 18:21:52 -0700 Subject: [PATCH 079/282] Avoid using wraps in dask_array_compat (#2053) This code led to a bug upstream in dask. The simplest choice is to remove it since we don't need docstrings for this function as used internally in xarray: https://github.com/dask/dask/issues/3388 --- xarray/core/dask_array_compat.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/xarray/core/dask_array_compat.py b/xarray/core/dask_array_compat.py index f6cb488aa4a..c2417345f55 100644 --- a/xarray/core/dask_array_compat.py +++ b/xarray/core/dask_array_compat.py @@ -1,6 +1,5 @@ from __future__ import absolute_import, division, print_function -from functools import wraps import numpy as np import dask.array as da @@ -15,7 +14,6 @@ def _isin_kernel(element, test_elements, assume_unique=False): assume_unique=assume_unique) return values.reshape(element.shape + (1,) * test_elements.ndim) - @wraps(getattr(np, 'isin', None)) def isin(element, test_elements, assume_unique=False, invert=False): element = da.asarray(element) test_elements = da.asarray(test_elements) From 4cbb7cbd86af1ccfe2b3b98f0e36a410f86d77ef Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 13 Apr 2018 18:37:49 -0700 Subject: [PATCH 080/282] Release v0.10.3 --- doc/whats-new.rst | 7 +++---- setup.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index efe8277e922..80f84d3d751 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -28,11 +28,10 @@ What's New .. _whats-new.0.10.3: -v0.10.3 (unreleased) --------------------- +v0.10.3 (April 13, 2018) +------------------------ -Documentation -~~~~~~~~~~~~~ +The minor release includes a number of bug-fixes and backwards compatible enhancements. Enhancements ~~~~~~~~~~~~ diff --git a/setup.py b/setup.py index d26c5c78dfc..2dde796ff29 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ MAJOR = 0 MINOR = 10 -MICRO = 2 -ISRELEASED = False +MICRO = 3 +ISRELEASED = True VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From 3dc7cbe4ab189c2eb17b3d482b5e6ee5d266405e Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 13 Apr 2018 18:40:38 -0700 Subject: [PATCH 081/282] Revert to v0.10.3 dev --- doc/whats-new.rst | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 80f84d3d751..e4007b0a7e9 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -26,6 +26,18 @@ What's New - `Tips on porting to Python 3 `__ +.. _whats-new.0.10.4: + +v0.10.4 (unreleased) +-------------------- + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + + .. _whats-new.0.10.3: v0.10.3 (April 13, 2018) diff --git a/setup.py b/setup.py index 2dde796ff29..c7c02c90e2f 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ MAJOR = 0 MINOR = 10 MICRO = 3 -ISRELEASED = True +ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From a0bdbfbe5e2333d150930807e3c31f33ab455d26 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 15 Apr 2018 21:07:58 -0400 Subject: [PATCH 082/282] Updates for the renaming of netcdftime to cftime (#2054) * Update for renaming of netcdftime to cftime * Use pip to install cftime for now * Update documentation * Use now-available conda-forge installation pathway --- ci/requirements-py27-cdat+iris+pynio.yml | 2 +- ci/requirements-py27-windows.yml | 2 +- ci/requirements-py35.yml | 2 +- ci/requirements-py36-bottleneck-dev.yml | 2 +- ci/requirements-py36-condaforge-rc.yml | 2 +- ci/requirements-py36-dask-dev.yml | 2 +- ci/requirements-py36-netcdf4-dev.yml | 2 +- ci/requirements-py36-pandas-dev.yml | 2 +- ci/requirements-py36-pynio-dev.yml | 2 +- ci/requirements-py36-rasterio1.0alpha.yml | 2 +- ci/requirements-py36-windows.yml | 3 +- ci/requirements-py36-zarr-dev.yml | 2 +- ci/requirements-py36.yml | 2 +- doc/installing.rst | 2 +- doc/time-series.rst | 8 ++-- xarray/coding/times.py | 32 ++++++-------- xarray/tests/__init__.py | 5 ++- xarray/tests/test_coding_times.py | 52 +++++++++++------------ xarray/tests/test_conventions.py | 10 ++--- 19 files changed, 68 insertions(+), 68 deletions(-) diff --git a/ci/requirements-py27-cdat+iris+pynio.yml b/ci/requirements-py27-cdat+iris+pynio.yml index 119d4eb0ffc..116e323d517 100644 --- a/ci/requirements-py27-cdat+iris+pynio.yml +++ b/ci/requirements-py27-cdat+iris+pynio.yml @@ -4,6 +4,7 @@ channels: dependencies: - python=2.7 - cdat-lite + - cftime - cyordereddict - dask - distributed @@ -11,7 +12,6 @@ dependencies: - h5netcdf - matplotlib - netcdf4 - - netcdftime - numpy - pandas - pathlib2 diff --git a/ci/requirements-py27-windows.yml b/ci/requirements-py27-windows.yml index 32aa0da0e96..43b292100de 100644 --- a/ci/requirements-py27-windows.yml +++ b/ci/requirements-py27-windows.yml @@ -3,13 +3,13 @@ channels: - conda-forge dependencies: - python=2.7 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pathlib2 - pytest - flake8 diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index 10c0eaead95..d3500bc5d10 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -3,13 +3,13 @@ channels: - conda-forge dependencies: - python=3.5 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-bottleneck-dev.yml b/ci/requirements-py36-bottleneck-dev.yml index 2d7a715ee44..b8619658929 100644 --- a/ci/requirements-py36-bottleneck-dev.yml +++ b/ci/requirements-py36-bottleneck-dev.yml @@ -3,13 +3,13 @@ channels: - conda-forge dependencies: - python=3.6 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-condaforge-rc.yml b/ci/requirements-py36-condaforge-rc.yml index 7e61514b877..8436d4e3e83 100644 --- a/ci/requirements-py36-condaforge-rc.yml +++ b/ci/requirements-py36-condaforge-rc.yml @@ -4,13 +4,13 @@ channels: - conda-forge dependencies: - python=3.6 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-dask-dev.yml b/ci/requirements-py36-dask-dev.yml index 4bffce496f9..54cdb54e8fc 100644 --- a/ci/requirements-py36-dask-dev.yml +++ b/ci/requirements-py36-dask-dev.yml @@ -3,11 +3,11 @@ channels: - conda-forge dependencies: - python=3.6 + - cftime - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-netcdf4-dev.yml b/ci/requirements-py36-netcdf4-dev.yml index 7d046743e7e..a473ceb5b0a 100644 --- a/ci/requirements-py36-netcdf4-dev.yml +++ b/ci/requirements-py36-netcdf4-dev.yml @@ -19,4 +19,4 @@ dependencies: - coveralls - pytest-cov - git+https://github.com/Unidata/netcdf4-python.git - - git+https://github.com/Unidata/netcdftime.git + - git+https://github.com/Unidata/cftime.git diff --git a/ci/requirements-py36-pandas-dev.yml b/ci/requirements-py36-pandas-dev.yml index 6ecd1c47f0d..1f1acabcae9 100644 --- a/ci/requirements-py36-pandas-dev.yml +++ b/ci/requirements-py36-pandas-dev.yml @@ -3,6 +3,7 @@ channels: - conda-forge dependencies: - python=3.6 + - cftime - cython - dask - distributed @@ -10,7 +11,6 @@ dependencies: - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - flake8 - numpy diff --git a/ci/requirements-py36-pynio-dev.yml b/ci/requirements-py36-pynio-dev.yml index 4667a66444b..75fbad900c7 100644 --- a/ci/requirements-py36-pynio-dev.yml +++ b/ci/requirements-py36-pynio-dev.yml @@ -4,13 +4,13 @@ channels: - ncar dependencies: - python=3.6 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pynio=dev - pytest - numpy diff --git a/ci/requirements-py36-rasterio1.0alpha.yml b/ci/requirements-py36-rasterio1.0alpha.yml index 8a2942a4d08..15ba13e753b 100644 --- a/ci/requirements-py36-rasterio1.0alpha.yml +++ b/ci/requirements-py36-rasterio1.0alpha.yml @@ -4,13 +4,13 @@ channels: - conda-forge/label/dev dependencies: - python=3.6 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - numpy - pandas diff --git a/ci/requirements-py36-windows.yml b/ci/requirements-py36-windows.yml index 228ce285787..62f08318087 100644 --- a/ci/requirements-py36-windows.yml +++ b/ci/requirements-py36-windows.yml @@ -3,13 +3,13 @@ channels: - conda-forge dependencies: - python=3.6 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - numpy - pandas @@ -18,3 +18,4 @@ dependencies: - toolz - rasterio - zarr + diff --git a/ci/requirements-py36-zarr-dev.yml b/ci/requirements-py36-zarr-dev.yml index b5991ba4403..7fbce63aa81 100644 --- a/ci/requirements-py36-zarr-dev.yml +++ b/ci/requirements-py36-zarr-dev.yml @@ -3,6 +3,7 @@ channels: - conda-forge dependencies: - python=3.6 + - cftime - dask - distributed - matplotlib @@ -14,7 +15,6 @@ dependencies: - seaborn - toolz - bottleneck - - netcdftime - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index 2788221ee74..0790f20764d 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -3,13 +3,13 @@ channels: - conda-forge dependencies: - python=3.6 + - cftime - dask - distributed - h5py - h5netcdf - matplotlib - netcdf4 - - netcdftime - pytest - flake8 - numpy diff --git a/doc/installing.rst b/doc/installing.rst index df443cbfed5..bb42129deea 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -25,7 +25,7 @@ For netCDF and IO - `pynio `__: for reading GRIB and other geoscience specific file formats - `zarr `__: for chunked, compressed, N-dimensional arrays. -- `netcdftime `__: recommended if you +- `cftime `__: recommended if you want to encode/decode datetimes for non-standard calendars or dates before year 1678 or after year 2262. diff --git a/doc/time-series.rst b/doc/time-series.rst index acc10220f27..afd9f087bfe 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -50,12 +50,12 @@ attribute like ``'days since 2000-01-01'``). .. note:: When decoding/encoding datetimes for non-standard calendars or for dates - before year 1678 or after year 2262, xarray uses the `netcdftime`_ library. - ``netcdftime`` was previously packaged with the ``netcdf4-python`` package but - is now distributed separately. ``netcdftime`` is an + before year 1678 or after year 2262, xarray uses the `cftime`_ library. + It was previously packaged with the ``netcdf4-python`` package under the + name ``netcdftime`` but is now distributed separately. ``cftime`` is an :ref:`optional dependency` of xarray. -.. _netcdftime: https://unidata.github.io/netcdftime +.. _cftime: https://unidata.github.io/cftime You can manual decode arrays in this form by passing a dataset to diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 8a1e9f82c6c..0a48b62986e 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -38,24 +38,20 @@ 'milliseconds', 'microseconds']) -def _import_netcdftime(): +def _import_cftime(): ''' - helper function handle the transition to netcdftime as a stand-alone - package + helper function handle the transition to netcdftime/cftime + as a stand-alone package ''' try: - # Try importing netcdftime directly - import netcdftime as nctime - if not hasattr(nctime, 'num2date'): - # must have gotten an old version from netcdf4-python - raise ImportError + import cftime except ImportError: # in netCDF4 the num2date/date2num function are top-level api try: - import netCDF4 as nctime + import netCDF4 as cftime except ImportError: - raise ImportError("Failed to import netcdftime") - return nctime + raise ImportError("Failed to import cftime") + return cftime def _netcdf_to_numpy_timeunit(units): @@ -78,9 +74,9 @@ def _unpack_netcdf_time_units(units): def _decode_datetime_with_netcdftime(num_dates, units, calendar): - nctime = _import_netcdftime() + cftime = _import_cftime() - dates = np.asarray(nctime.num2date(num_dates, units, calendar)) + dates = np.asarray(cftime.num2date(num_dates, units, calendar)) if (dates[np.nanargmin(num_dates)].year < 1678 or dates[np.nanargmax(num_dates)].year >= 2262): warnings.warn('Unable to decode time axis into full ' @@ -89,7 +85,7 @@ def _decode_datetime_with_netcdftime(num_dates, units, calendar): ' of range', SerializationWarning, stacklevel=3) else: try: - dates = nctime_to_nptime(dates) + dates = cftime_to_nptime(dates) except ValueError as e: warnings.warn('Unable to decode time axis into full ' 'numpy.datetime64 objects, continuing using ' @@ -232,8 +228,8 @@ def infer_timedelta_units(deltas): return units -def nctime_to_nptime(times): - """Given an array of netcdftime.datetime objects, return an array of +def cftime_to_nptime(times): + """Given an array of cftime.datetime objects, return an array of numpy.datetime64 objects of the same size""" times = np.asarray(times) new = np.empty(times.shape, dtype='M8[ns]') @@ -259,14 +255,14 @@ def _encode_datetime_with_netcdftime(dates, units, calendar): This method is more flexible than xarray's parsing using datetime64[ns] arrays but also slower because it loops over each element. """ - nctime = _import_netcdftime() + cftime = _import_cftime() if np.issubdtype(dates.dtype, np.datetime64): # numpy's broken datetime conversion only works for us precision dates = dates.astype('M8[us]').astype(datetime) def encode_datetime(d): - return np.nan if d is None else nctime.date2num(d, units, calendar) + return np.nan if d is None else cftime.date2num(d, units, calendar) return np.vectorize(encode_datetime)(dates) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 7c9528d741d..7584ed79a06 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -68,7 +68,7 @@ def _importorskip(modname, minversion=None): has_netCDF4, requires_netCDF4 = _importorskip('netCDF4') has_h5netcdf, requires_h5netcdf = _importorskip('h5netcdf') has_pynio, requires_pynio = _importorskip('Nio') -has_netcdftime, requires_netcdftime = _importorskip('netcdftime') +has_cftime, requires_cftime = _importorskip('cftime') has_dask, requires_dask = _importorskip('dask') has_bottleneck, requires_bottleneck = _importorskip('bottleneck') has_rasterio, requires_rasterio = _importorskip('rasterio') @@ -80,6 +80,9 @@ def _importorskip(modname, minversion=None): has_scipy_or_netCDF4 = has_scipy or has_netCDF4 requires_scipy_or_netCDF4 = unittest.skipUnless( has_scipy_or_netCDF4, reason='requires scipy or netCDF4') +has_cftime_or_netCDF4 = has_cftime or has_netCDF4 +requires_cftime_or_netCDF4 = unittest.skipUnless( + has_cftime_or_netCDF4, reason='requires cftime or netCDF4') if not has_pathlib: has_pathlib, requires_pathlib = _importorskip('pathlib2') if has_dask: diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index ab33329b51a..7e69d4b3ff2 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -7,9 +7,9 @@ import pytest from xarray import Variable, coding -from xarray.coding.times import _import_netcdftime +from xarray.coding.times import _import_cftime -from . import TestCase, assert_array_equal, requires_netcdftime +from . import TestCase, assert_array_equal, requires_cftime_or_netCDF4 @np.vectorize @@ -21,9 +21,9 @@ def _ensure_naive_tz(dt): class TestDatetime(TestCase): - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_cf_datetime(self): - nctime = _import_netcdftime() + cftime = _import_cftime() for num_dates, units in [ (np.arange(10), 'days since 2000-01-01'), (np.arange(10).astype('float64'), 'days since 2000-01-01'), @@ -53,7 +53,7 @@ def test_cf_datetime(self): ]: for calendar in ['standard', 'gregorian', 'proleptic_gregorian']: expected = _ensure_naive_tz( - nctime.num2date(num_dates, units, calendar)) + cftime.num2date(num_dates, units, calendar)) print(num_dates, units, calendar) with warnings.catch_warnings(): warnings.filterwarnings('ignore', @@ -88,7 +88,7 @@ def test_cf_datetime(self): pd.Index(actual), units, calendar) assert_array_equal(num_dates, np.around(encoded, 1)) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_cf_datetime_overflow(self): # checks for # https://github.com/pydata/pandas/issues/14068 @@ -113,7 +113,7 @@ def test_decode_cf_datetime_non_standard_units(self): actual = coding.times.decode_cf_datetime(np.arange(100), units) assert_array_equal(actual, expected) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_cf_datetime_non_iso_strings(self): # datetime strings that are _almost_ ISO compliant but not quite, # but which netCDF4.num2date can still parse correctly @@ -125,16 +125,16 @@ def test_decode_cf_datetime_non_iso_strings(self): actual = coding.times.decode_cf_datetime(num_dates, units) assert_array_equal(actual, expected) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_non_standard_calendar(self): - nctime = _import_netcdftime() + cftime = _import_cftime() for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', '366_day']: units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') - noleap_time = nctime.date2num(times.to_pydatetime(), units, + noleap_time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar) expected = times.values with warnings.catch_warnings(): @@ -148,7 +148,7 @@ def test_decode_non_standard_calendar(self): # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, 's')).all() - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_non_standard_calendar_single_element(self): units = 'days since 0001-01-01' for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', @@ -161,36 +161,36 @@ def test_decode_non_standard_calendar_single_element(self): calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_non_standard_calendar_single_element_fallback(self): - nctime = _import_netcdftime() + cftime = _import_cftime() units = 'days since 0001-01-01' try: - dt = nctime.netcdftime.datetime(2001, 2, 29) + dt = cftime.netcdftime.datetime(2001, 2, 29) except AttributeError: # Must be using standalone netcdftime library - dt = nctime.datetime(2001, 2, 29) + dt = cftime.datetime(2001, 2, 29) for calendar in ['360_day', 'all_leap', '366_day']: - num_time = nctime.date2num(dt, units, calendar) + num_time = cftime.date2num(dt, units, calendar) with pytest.warns(Warning, match='Unable to decode time axis'): actual = coding.times.decode_cf_datetime(num_time, units, calendar=calendar) - expected = np.asarray(nctime.num2date(num_time, units, calendar)) + expected = np.asarray(cftime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_non_standard_calendar_multidim_time(self): - nctime = _import_netcdftime() + cftime = _import_cftime() calendar = 'noleap' units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = nctime.date2num(times1.to_pydatetime(), units, + noleap_time1 = cftime.date2num(times1.to_pydatetime(), units, calendar=calendar) - noleap_time2 = nctime.date2num(times2.to_pydatetime(), units, + noleap_time2 = cftime.date2num(times2.to_pydatetime(), units, calendar=calendar) mdim_time = np.empty((len(noleap_time1), 2), ) mdim_time[:, 0] = noleap_time1 @@ -206,16 +206,16 @@ def test_decode_non_standard_calendar_multidim_time(self): assert_array_equal(actual[:, 0], expected1) assert_array_equal(actual[:, 1], expected2) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_non_standard_calendar_fallback(self): - nctime = _import_netcdftime() + cftime = _import_cftime() # ensure leap year doesn't matter for year in [2010, 2011, 2012, 2013, 2014]: for calendar in ['360_day', '366_day', 'all_leap']: calendar = '360_day' units = 'days since {0}-01-01'.format(year) num_times = np.arange(100) - expected = nctime.num2date(num_times, units, calendar) + expected = cftime.num2date(num_times, units, calendar) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') @@ -228,7 +228,7 @@ def test_decode_non_standard_calendar_fallback(self): assert actual.dtype == np.dtype('O') assert_array_equal(actual, expected) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_cf_datetime_nan(self): for num_dates, units, expected_list in [ ([np.nan], 'days since 2000-01-01', ['NaT']), @@ -243,7 +243,7 @@ def test_cf_datetime_nan(self): expected = np.array(expected_list, dtype='datetime64[ns]') assert_array_equal(expected, actual) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decoded_cf_datetime_array_2d(self): # regression test for GH1229 variable = Variable(('x', 'y'), np.array([[0, 1], [2, 3]]), diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 86240f4e33e..80c69ece5dd 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -18,7 +18,7 @@ from . import ( IndexerMaker, TestCase, assert_array_equal, raises_regex, requires_netCDF4, - requires_netcdftime, unittest, requires_dask) + requires_cftime_or_netCDF4, unittest, requires_dask) from .test_backends import CFEncodedDataTest B = IndexerMaker(indexing.BasicIndexer) @@ -183,7 +183,7 @@ def test_decode_cf_with_conflicting_fill_missing_value(): assert_identical(actual, expected) -@requires_netcdftime +@requires_cftime_or_netCDF4 class TestEncodeCFVariable(TestCase): def test_incompatible_attributes(self): invalid_vars = [ @@ -239,7 +239,7 @@ def test_multidimensional_coordinates(self): assert 'coordinates' not in attrs -@requires_netcdftime +@requires_cftime_or_netCDF4 class TestDecodeCF(TestCase): def test_dataset(self): original = Dataset({ @@ -305,7 +305,7 @@ def test_invalid_time_units_raises_eagerly(self): with raises_regex(ValueError, 'unable to decode time'): decode_cf(ds) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_dataset_repr_with_netcdf4_datetimes(self): # regression test for #347 attrs = {'units': 'days since 0001-01-01', 'calendar': 'noleap'} @@ -318,7 +318,7 @@ def test_dataset_repr_with_netcdf4_datetimes(self): ds = decode_cf(Dataset({'time': ('time', [0, 1], attrs)})) assert '(time) datetime64[ns]' in repr(ds) - @requires_netcdftime + @requires_cftime_or_netCDF4 def test_decode_cf_datetime_transition_to_invalid(self): # manually create dataset with not-decoded date from datetime import datetime From 68090bbfc6ce3ea72f8166375441d31f830de1ea Mon Sep 17 00:00:00 2001 From: Dan Nowacki Date: Tue, 17 Apr 2018 11:39:33 -0400 Subject: [PATCH 083/282] ENH: setncattr_string support (GH2044) (#2045) --- doc/whats-new.rst | 3 +++ xarray/backends/netCDF4_.py | 29 +++++++++++++++++++++++++++-- xarray/tests/test_backends.py | 17 +++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e4007b0a7e9..d042e1df1e9 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -34,6 +34,9 @@ v0.10.4 (unreleased) Enhancements ~~~~~~~~~~~~ +- Support writing lists of strings as netCDF attributes (:issue:`2044`). + By `Dan Nowacki `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 89a0e72ef07..be714082d3b 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -229,6 +229,31 @@ def _disable_auto_decode_group(ds): _disable_auto_decode_variable(var) +def _is_list_of_strings(value): + if (np.asarray(value).dtype.kind in ['U', 'S'] and + np.asarray(value).size > 1): + return True + else: + return False + + +def _set_nc_attribute(obj, key, value): + if _is_list_of_strings(value): + # encode as NC_STRING if attr is list of strings + try: + obj.setncattr_string(key, value) + except AttributeError: + # Inform users with old netCDF that does not support + # NC_STRING that we can't serialize lists of strings + # as attrs + msg = ('Attributes which are lists of strings are not ' + 'supported with this version of netCDF. Please ' + 'upgrade to netCDF4-python 1.2.4 or greater.') + raise AttributeError(msg) + else: + obj.setncattr(key, value) + + class NetCDF4DataStore(WritableCFDataStore, DataStorePickleMixin): """Store for reading and writing data via the Python-NetCDF4 library. @@ -347,7 +372,7 @@ def set_attribute(self, key, value): with self.ensure_open(autoclose=False): if self.format != 'NETCDF4': value = encode_nc3_attr_value(value) - self.ds.setncattr(key, value) + _set_nc_attribute(self.ds, key, value) def set_variables(self, *args, **kwargs): with self.ensure_open(autoclose=False): @@ -402,7 +427,7 @@ def prepare_variable(self, name, variable, check_encoding=False, for k, v in iteritems(attrs): # set attributes one-by-one since netCDF4<1.0.10 can't handle # OrderedDict as the input to setncatts - nc4_var.setncattr(k, v) + _set_nc_attribute(nc4_var, k, v) target = NetCDF4ArrayWrapper(name, self) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index c5e8d126e43..8b20f6148e7 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1144,6 +1144,23 @@ def test_88_character_filename_segmentation_fault(self): # Need to construct 88 character filepath xr.Dataset().to_netcdf('a' * (88 - len(os.getcwd()) - 1)) + def test_setncattr_string(self): + list_of_strings = ['list', 'of', 'strings'] + one_element_list_of_strings = ['one element'] + one_string = 'one string' + attrs = {'foo': list_of_strings, + 'bar': one_element_list_of_strings, + 'baz': one_string} + ds = Dataset({'x': ('y', [1, 2, 3], attrs)}, + attrs=attrs) + + with self.roundtrip(ds) as actual: + for totest in [actual, actual['x']]: + assert_array_equal(list_of_strings, totest.attrs['foo']) + assert_array_equal(one_element_list_of_strings, + totest.attrs['bar']) + assert one_string == totest.attrs['baz'] + class NetCDF4DataStoreAutocloseTrue(NetCDF4DataTest): autoclose = True From 093518207f4a3210ec692308da2b181a646115d6 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 20 Apr 2018 13:04:22 +0100 Subject: [PATCH 084/282] Parallel open_mfdataset (#1983) * proof of concept implementation for parallel open using dask.bag * parallel open option in open_mfdataset * use dask delayed, further tests to check that dask arrays make it through * parallel=None by default and automagicalize to True when dask distributed is used * docs and minor improvement to delayed/compute * 500 file max * skip many-files-test on windows * cleanup parallel open a bit * refactor parallel open a bit, fix tests and fixtures * default parallel open when dask is installed + style fixes * fix parallel kwarg * pynio from conda-forge/dev for pynio test suite * manually skip tests * parallel defaults to false --- ci/requirements-py36-pynio-dev.yml | 2 +- doc/whats-new.rst | 8 +- xarray/backends/api.py | 31 +++++-- xarray/tests/test_backends.py | 136 +++++++++++++---------------- 4 files changed, 94 insertions(+), 83 deletions(-) diff --git a/ci/requirements-py36-pynio-dev.yml b/ci/requirements-py36-pynio-dev.yml index 75fbad900c7..2caaa8affe5 100644 --- a/ci/requirements-py36-pynio-dev.yml +++ b/ci/requirements-py36-pynio-dev.yml @@ -1,7 +1,7 @@ name: test_env channels: - conda-forge - - ncar + - conda-forge/label/dev dependencies: - python=3.6 - cftime diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d042e1df1e9..88a207a2f80 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -51,16 +51,20 @@ The minor release includes a number of bug-fixes and backwards compatible enhanc Enhancements ~~~~~~~~~~~~ +- Added the ``parallel`` option to :py:func:`open_mfdataset`. This option uses + ``dask.delayed`` to parallelize the open and preprocessing steps within + ``open_mfdataset``. This is expected to provide performance improvements when + opening many files, particularly when used in conjunction with dask's + multiprocessing or distributed schedulers (:issue:`1981`). + By `Joe Hamman `_. - :py:meth:`~xarray.DataArray.isin` and :py:meth:`~xarray.Dataset.isin` methods, which test each value in the array for whether it is contained in the supplied list, returning a bool array. See :ref:`selecting values with isin` for full details. Similar to the ``np.isin`` function. By `Maximilian Roos `_. - - Some speed improvement to construct :py:class:`~xarray.DataArrayRolling` object (:issue:`1993`) By `Keisuke Fujii `_. - - Handle variables with different values for ``missing_value`` and ``_FillValue`` by masking values for both attributes; previously this resulted in a ``ValueError``. (:issue:`2016`) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index a22356f66b0..a3fa753c6d9 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -453,7 +453,8 @@ def close(self): def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, compat='no_conflicts', preprocess=None, engine=None, - lock=None, data_vars='all', coords='different', **kwargs): + lock=None, data_vars='all', coords='different', + autoclose=False, parallel=False, **kwargs): """Open multiple files as a single dataset. Requires dask to be installed. See documentation for details on dask [1]. @@ -534,7 +535,9 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, those corresponding to other dimensions. * list of str: The listed coordinate variables will be concatenated, in addition the 'minimal' coordinates. - + parallel : bool, optional + If True, the open and preprocess steps of this function will be + performed in parallel using ``dask.delayed``. Default is False. **kwargs : optional Additional arguments passed on to :py:func:`xarray.open_dataset`. @@ -562,13 +565,31 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, if lock is None: lock = _default_lock(paths[0], engine) - datasets = [open_dataset(p, engine=engine, chunks=chunks or {}, lock=lock, - **kwargs) for p in paths] - file_objs = [ds._file_obj for ds in datasets] + open_kwargs = dict(engine=engine, chunks=chunks or {}, lock=lock, + autoclose=autoclose, **kwargs) + + if parallel: + import dask + # wrap the open_dataset, getattr, and preprocess with delayed + open_ = dask.delayed(open_dataset) + getattr_ = dask.delayed(getattr) + if preprocess is not None: + preprocess = dask.delayed(preprocess) + else: + open_ = open_dataset + getattr_ = getattr + + datasets = [open_(p, **open_kwargs) for p in paths] + file_objs = [getattr_(ds, '_file_obj') for ds in datasets] if preprocess is not None: datasets = [preprocess(ds) for ds in datasets] + if parallel: + # calling compute here will return the datasets/file_objs lists, + # the underlying datasets will still be stored as dask arrays + datasets, file_objs = dask.compute(datasets, file_objs) + # close datasets in case of a ValueError try: if concat_dim is _CONCAT_DIM_DEFAULT: diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 8b20f6148e7..c7b53e5ecde 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -29,7 +29,7 @@ from . import ( TestCase, assert_allclose, assert_array_equal, assert_equal, - assert_identical, flaky, has_netCDF4, has_scipy, network, raises_regex, + assert_identical, has_dask, has_netCDF4, has_scipy, network, raises_regex, requires_dask, requires_h5netcdf, requires_netCDF4, requires_pathlib, requires_pydap, requires_pynio, requires_rasterio, requires_scipy, requires_scipy_or_netCDF4, requires_zarr) @@ -1693,88 +1693,74 @@ class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest): autoclose = True -class OpenMFDatasetManyFilesTest(TestCase): - def validate_open_mfdataset_autoclose(self, engine, nfiles=10): - randdata = np.random.randn(nfiles) - original = Dataset({'foo': ('x', randdata)}) - # test standard open_mfdataset approach with too many files - with create_tmp_files(nfiles) as tmpfiles: - for readengine in engine: - writeengine = (readengine if readengine != 'pynio' - else 'netcdf4') - # split into multiple sets of temp files - for ii in original.x.values: - subds = original.isel(x=slice(ii, ii + 1)) - subds.to_netcdf(tmpfiles[ii], engine=writeengine) - - # check that calculation on opened datasets works properly - ds = open_mfdataset(tmpfiles, engine=readengine, - autoclose=True) - self.assertAllClose(ds.x.sum().values, - (nfiles * (nfiles - 1)) / 2) - self.assertAllClose(ds.foo.sum().values, np.sum(randdata)) - self.assertAllClose(ds.sum().foo.values, np.sum(randdata)) - ds.close() - - def validate_open_mfdataset_large_num_files(self, engine): - self.validate_open_mfdataset_autoclose(engine, nfiles=2000) +@pytest.fixture(params=['scipy', 'netcdf4', 'h5netcdf', 'pynio']) +def readengine(request): + return request.param - @requires_dask - @requires_netCDF4 - def test_1_autoclose_netcdf4(self): - self.validate_open_mfdataset_autoclose(engine=['netcdf4']) - @requires_dask - @requires_scipy - def test_2_autoclose_scipy(self): - self.validate_open_mfdataset_autoclose(engine=['scipy']) +@pytest.fixture(params=[1, 100]) +def nfiles(request): + return request.param - @requires_dask - @requires_pynio - def test_3_autoclose_pynio(self): - self.validate_open_mfdataset_autoclose(engine=['pynio']) - # use of autoclose=True with h5netcdf broken because of - # probable h5netcdf error - @requires_dask - @requires_h5netcdf - @pytest.mark.xfail - def test_4_autoclose_h5netcdf(self): - self.validate_open_mfdataset_autoclose(engine=['h5netcdf']) +@pytest.fixture(params=[True, False]) +def autoclose(request): + return request.param - # These tests below are marked as flaky (and skipped by default) because - # they fail sometimes on Travis-CI, for no clear reason. - @requires_dask - @requires_netCDF4 - @flaky - @pytest.mark.slow - def test_1_open_large_num_files_netcdf4(self): - self.validate_open_mfdataset_large_num_files(engine=['netcdf4']) +@pytest.fixture(params=[True, False]) +def parallel(request): + return request.param - @requires_dask - @requires_scipy - @flaky - @pytest.mark.slow - def test_2_open_large_num_files_scipy(self): - self.validate_open_mfdataset_large_num_files(engine=['scipy']) - @requires_dask - @requires_pynio - @flaky - @pytest.mark.slow - def test_3_open_large_num_files_pynio(self): - self.validate_open_mfdataset_large_num_files(engine=['pynio']) - - # use of autoclose=True with h5netcdf broken because of - # probable h5netcdf error - @requires_dask - @requires_h5netcdf - @flaky - @pytest.mark.xfail - @pytest.mark.slow - def test_4_open_large_num_files_h5netcdf(self): - self.validate_open_mfdataset_large_num_files(engine=['h5netcdf']) +@pytest.fixture(params=[None, 5]) +def chunks(request): + return request.param + + +# using pytest.mark.skipif does not work so this a work around +def skip_if_not_engine(engine): + if engine == 'netcdf4': + pytest.importorskip('netCDF4') + elif engine == 'pynio': + pytest.importorskip('Nio') + else: + pytest.importorskip(engine) + + +def test_open_mfdataset_manyfiles(readengine, nfiles, autoclose, parallel, + chunks): + + # skip certain combinations + skip_if_not_engine(readengine) + + if not has_dask and parallel: + pytest.skip('parallel requires dask') + + if readengine == 'h5netcdf' and autoclose: + pytest.skip('h5netcdf does not support autoclose yet') + + if ON_WINDOWS: + pytest.skip('Skipping on Windows') + + randdata = np.random.randn(nfiles) + original = Dataset({'foo': ('x', randdata)}) + # test standard open_mfdataset approach with too many files + with create_tmp_files(nfiles) as tmpfiles: + writeengine = (readengine if readengine != 'pynio' else 'netcdf4') + # split into multiple sets of temp files + for ii in original.x.values: + subds = original.isel(x=slice(ii, ii + 1)) + subds.to_netcdf(tmpfiles[ii], engine=writeengine) + + # check that calculation on opened datasets works properly + actual = open_mfdataset(tmpfiles, engine=readengine, parallel=parallel, + autoclose=autoclose, chunks=chunks) + + # check that using open_mfdataset returns dask arrays for variables + assert isinstance(actual['foo'].data, dask_array_type) + + assert_identical(original, actual) @requires_scipy_or_netCDF4 From 99b457ce5859bd949cfea4671db5150c7297843a Mon Sep 17 00:00:00 2001 From: Michael Delgado Date: Sat, 21 Apr 2018 10:42:06 -0700 Subject: [PATCH 085/282] resolve #2071: 'bebroadcast' in ValueError msg (#2072) Add space to end of line halfway through error message on line 682 to avoid 'bebroadcast' on string concatenation. --- xarray/core/variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 5ec85c159a2..622ac60d7f6 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -679,7 +679,7 @@ def __setitem__(self, key, value): value = as_compatible_data(value) if value.ndim > len(dims): raise ValueError( - 'shape mismatch: value array of shape %s could not be' + 'shape mismatch: value array of shape %s could not be ' 'broadcast to indexing result with %s dimensions' % (value.shape, len(dims))) if value.ndim == 0: From c42cbe787d530db48575d73fe7a3910b0a0e4fd8 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 26 Apr 2018 08:48:22 -0700 Subject: [PATCH 086/282] Better error handling in open_mfdataset for wildcard remote URLs (#2083) Fixes GH2077 --- doc/whats-new.rst | 2 ++ xarray/backends/api.py | 5 +++++ xarray/tests/test_backends.py | 3 +++ 3 files changed, 10 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 88a207a2f80..d804fe7b915 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -40,6 +40,8 @@ Enhancements Bug fixes ~~~~~~~~~ +- Better error handling in ``open_mfdataset`` (:issue:`2077`). + By `Stephan Hoyer `_. .. _whats-new.0.10.3: diff --git a/xarray/backends/api.py b/xarray/backends/api.py index a3fa753c6d9..da4ef537a3a 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -556,6 +556,11 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, .. [2] http://xarray.pydata.org/en/stable/dask.html#chunking-and-performance """ if isinstance(paths, basestring): + if is_remote_uri(paths): + raise ValueError( + 'cannot do wild-card matching for paths that are remote URLs: ' + '{!r}. Instead, supply paths as an explicit list of strings.' + .format(paths)) paths = sorted(glob(paths)) else: paths = [str(p) if isinstance(p, path_type) else p for p in paths] diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index c7b53e5ecde..1e69363225f 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1928,6 +1928,9 @@ def test_open_mfdataset(self): with raises_regex(IOError, 'no files to open'): open_mfdataset('foo-bar-baz-*.nc', autoclose=self.autoclose) + with raises_regex(ValueError, 'wild-card'): + open_mfdataset('http://some/remote/uri', autoclose=self.autoclose) + @requires_pathlib def test_open_mfdataset_pathlib(self): original = Dataset({'foo': ('x', np.random.randn(10))}) From 3c8935e537e6ec05a83dbe372bfe45d88308d665 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 30 Apr 2018 10:17:26 -0700 Subject: [PATCH 087/282] Refactor string coding and fix string support in zarr (#2058) * Move string coding from conventions to xarray.coding. This refactor will make it easier to use different coding for different backends. It includes using a special dtype with metadata (like h5py) to identify variable length strings. So in principle, we could now handle fixed width and variable length unicode differently. * Zarr backend roundtrips strings and bytes * Add test validating that cf_decode() preserves dask string arrays. * Fix failing test on python 2.7 * typo --- xarray/backends/memory.py | 3 - xarray/backends/netCDF4_.py | 17 +- xarray/backends/netcdf3.py | 6 +- xarray/backends/zarr.py | 41 +--- xarray/coding/strings.py | 219 ++++++++++++++++++++++ xarray/coding/variables.py | 2 +- xarray/conventions.py | 279 ++++++---------------------- xarray/tests/test_backends.py | 18 -- xarray/tests/test_coding_strings.py | 218 ++++++++++++++++++++++ xarray/tests/test_conventions.py | 171 +++-------------- 10 files changed, 542 insertions(+), 432 deletions(-) create mode 100644 xarray/coding/strings.py create mode 100644 xarray/tests/test_coding_strings.py diff --git a/xarray/backends/memory.py b/xarray/backends/memory.py index 69a54133716..dcf092557b8 100644 --- a/xarray/backends/memory.py +++ b/xarray/backends/memory.py @@ -37,9 +37,6 @@ def get_dimensions(self): def prepare_variable(self, k, v, *args, **kwargs): new_var = Variable(v.dims, np.empty_like(v), v.attrs) - # we copy the variable and stuff all encodings in the - # attributes to imitate what happens when writing to disk. - new_var.attrs.update(v.encoding) self._variables[k] = new_var return new_var, v.data diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index be714082d3b..1195301825b 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -7,10 +7,11 @@ import numpy as np -from .. import Variable, conventions -from ..conventions import pop_to +from .. import Variable, coding +from ..coding.variables import pop_to from ..core import indexing -from ..core.pycompat import PY3, OrderedDict, basestring, iteritems, suppress +from ..core.pycompat import ( + PY3, OrderedDict, basestring, iteritems, suppress) from ..core.utils import FrozenOrderedDict, close_on_error, is_remote_uri from .common import ( HDF5_LOCK, BackendArray, DataStorePickleMixin, WritableCFDataStore, @@ -82,8 +83,9 @@ def __getitem__(self, key): def _encode_nc4_variable(var): - if var.dtype.kind == 'S': - var = conventions.maybe_encode_as_char_array(var) + for coder in [coding.strings.EncodedStringCoder(allows_unicode=True), + coding.strings.CharacterArrayCoder()]: + var = coder.encode(var) return var @@ -96,12 +98,13 @@ def _get_datatype(var, nc_format='NETCDF4'): def _nc4_dtype(var): - if var.dtype.kind == 'U': + if coding.strings.is_unicode_dtype(var.dtype): dtype = str elif var.dtype.kind in ['i', 'u', 'f', 'c', 'S']: dtype = var.dtype else: - raise ValueError('cannot infer dtype for netCDF4 variable') + raise ValueError('unsupported dtype for netCDF4 variable: {}' + .format(var.dtype)) return dtype diff --git a/xarray/backends/netcdf3.py b/xarray/backends/netcdf3.py index f0ded98d954..c7bfa0ea20b 100644 --- a/xarray/backends/netcdf3.py +++ b/xarray/backends/netcdf3.py @@ -4,7 +4,7 @@ import numpy as np -from .. import Variable, conventions +from .. import Variable, coding from ..core.pycompat import OrderedDict, basestring, unicode_type # Special characters that are permitted in netCDF names except in the @@ -65,7 +65,9 @@ def encode_nc3_attrs(attrs): def encode_nc3_variable(var): - var = conventions.maybe_encode_as_char_array(var) + for coder in [coding.strings.EncodedStringCoder(allows_unicode=False), + coding.strings.CharacterArrayCoder()]: + var = coder.encode(var) data = coerce_nc3_dtype(var.data) attrs = encode_nc3_attrs(var.attrs) return Variable(var.dims, data, attrs, var.encoding) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 71ce965f368..83dcbd9a172 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1,6 +1,5 @@ from __future__ import absolute_import, division, print_function -from base64 import b64encode from itertools import product from distutils.version import LooseVersion @@ -25,23 +24,11 @@ def _encode_zarr_attr_value(value): # this checks if it's a scalar number elif isinstance(value, np.generic): encoded = value.item() - # np.string_('X').item() returns a type `bytes` - # zarr still doesn't like that - if type(encoded) is bytes: # noqa - encoded = b64encode(encoded) else: encoded = value return encoded -def _ensure_valid_fill_value(value, dtype): - if dtype.type == np.string_ and type(value) == bytes: # noqa - valid = b64encode(value) - else: - valid = value - return _encode_zarr_attr_value(valid) - - class ZarrArrayWrapper(BackendArray): def __init__(self, variable_name, datastore): self.datastore = datastore @@ -221,22 +208,15 @@ def encode_zarr_variable(var, needs_copy=True, name=None): A variable which has been encoded as described above. """ - if var.dtype.kind == 'O': - raise NotImplementedError("Variable `%s` is an object. Zarr " - "store can't yet encode objects." % name) - - for coder in [coding.times.CFDatetimeCoder(), - coding.times.CFTimedeltaCoder(), - coding.variables.CFScaleOffsetCoder(), - coding.variables.CFMaskCoder(), - coding.variables.UnsignedIntegerCoder()]: - var = coder.encode(var, name=name) - - var = conventions.maybe_encode_nonstring_dtype(var, name=name) - var = conventions.maybe_default_fill_value(var) - var = conventions.maybe_encode_bools(var) - var = conventions.ensure_dtype_not_object(var, name=name) - var = conventions.maybe_encode_string_dtype(var, name=name) + var = conventions.encode_cf_variable(var, name=name) + + # zarr allows unicode, but not variable-length strings, so it's both + # simpler and more compact to always encode as UTF-8 explicitly. + # TODO: allow toggling this explicitly via dtype in encoding. + coder = coding.strings.EncodedStringCoder(allows_unicode=False) + var = coder.encode(var, name=name) + var = coding.strings.ensure_fixed_length_bytes(var) + return var @@ -339,8 +319,7 @@ def prepare_variable(self, name, variable, check_encoding=False, dtype = variable.dtype shape = variable.shape - fill_value = _ensure_valid_fill_value(attrs.pop('_FillValue', None), - dtype) + fill_value = attrs.pop('_FillValue', None) if variable.encoding == {'_FillValue': None} and fill_value is None: variable.encoding = {} diff --git a/xarray/coding/strings.py b/xarray/coding/strings.py new file mode 100644 index 00000000000..08edeed4153 --- /dev/null +++ b/xarray/coding/strings.py @@ -0,0 +1,219 @@ +"""Coders for strings.""" +from __future__ import absolute_import, division, print_function + +from functools import partial + +import numpy as np + +from ..core import indexing +from ..core.pycompat import bytes_type, dask_array_type, unicode_type +from ..core.variable import Variable +from .variables import ( + VariableCoder, lazy_elemwise_func, pop_to, + safe_setitem, unpack_for_decoding, unpack_for_encoding) + + +def create_vlen_dtype(element_type): + # based on h5py.special_dtype + return np.dtype('O', metadata={'element_type': element_type}) + + +def check_vlen_dtype(dtype): + if dtype.kind != 'O' or dtype.metadata is None: + return None + else: + return dtype.metadata.get('element_type') + + +def is_unicode_dtype(dtype): + return dtype.kind == 'U' or check_vlen_dtype(dtype) == unicode_type + + +def is_bytes_dtype(dtype): + return dtype.kind == 'S' or check_vlen_dtype(dtype) == bytes_type + + +class EncodedStringCoder(VariableCoder): + """Transforms between unicode strings and fixed-width UTF-8 bytes.""" + + def __init__(self, allows_unicode=True): + self.allows_unicode = allows_unicode + + def encode(self, variable, name=None): + dims, data, attrs, encoding = unpack_for_encoding(variable) + + contains_unicode = is_unicode_dtype(data.dtype) + encode_as_char = 'dtype' in encoding and encoding['dtype'] == 'S1' + + if contains_unicode and (encode_as_char or not self.allows_unicode): + if '_FillValue' in attrs: + raise NotImplementedError( + 'variable {!r} has a _FillValue specified, but ' + '_FillValue is not yet supported on unicode strings: ' + 'https://github.com/pydata/xarray/issues/1647' + .format(name)) + + string_encoding = encoding.pop('_Encoding', 'utf-8') + safe_setitem(attrs, '_Encoding', string_encoding, name=name) + # TODO: figure out how to handle this in a lazy way with dask + data = encode_string_array(data, string_encoding) + + return Variable(dims, data, attrs, encoding) + + def decode(self, variable, name=None): + dims, data, attrs, encoding = unpack_for_decoding(variable) + + if '_Encoding' in attrs: + string_encoding = pop_to(attrs, encoding, '_Encoding') + func = partial(decode_bytes_array, encoding=string_encoding) + data = lazy_elemwise_func(data, func, np.dtype(object)) + + return Variable(dims, data, attrs, encoding) + + +def decode_bytes_array(bytes_array, encoding='utf-8'): + # This is faster than using np.char.decode() or np.vectorize() + bytes_array = np.asarray(bytes_array) + decoded = [x.decode(encoding) for x in bytes_array.ravel()] + return np.array(decoded, dtype=object).reshape(bytes_array.shape) + + +def encode_string_array(string_array, encoding='utf-8'): + string_array = np.asarray(string_array) + encoded = [x.encode(encoding) for x in string_array.ravel()] + return np.array(encoded, dtype=bytes).reshape(string_array.shape) + + +def ensure_fixed_length_bytes(var): + """Ensure that a variable with vlen bytes is converted to fixed width.""" + dims, data, attrs, encoding = unpack_for_encoding(var) + if check_vlen_dtype(data.dtype) == bytes_type: + # TODO: figure out how to handle this with dask + data = np.asarray(data, dtype=np.string_) + return Variable(dims, data, attrs, encoding) + + +class CharacterArrayCoder(VariableCoder): + """Transforms between arrays containing bytes and character arrays.""" + + def encode(self, variable, name=None): + variable = ensure_fixed_length_bytes(variable) + + dims, data, attrs, encoding = unpack_for_encoding(variable) + if data.dtype.kind == 'S': + data = bytes_to_char(data) + dims = dims + ('string%s' % data.shape[-1],) + return Variable(dims, data, attrs, encoding) + + def decode(self, variable, name=None): + dims, data, attrs, encoding = unpack_for_decoding(variable) + + if data.dtype == 'S1' and dims: + dims = dims[:-1] + data = char_to_bytes(data) + + return Variable(dims, data, attrs, encoding) + + +def bytes_to_char(arr): + """Convert numpy/dask arrays from fixed width bytes to characters.""" + if arr.dtype.kind != 'S': + raise ValueError('argument must have a fixed-width bytes dtype') + + if isinstance(arr, dask_array_type): + import dask.array as da + return da.map_blocks(_numpy_bytes_to_char, arr, + dtype='S1', + chunks=arr.chunks + ((arr.dtype.itemsize,)), + new_axis=[arr.ndim]) + else: + return _numpy_bytes_to_char(arr) + + +def _numpy_bytes_to_char(arr): + """Like netCDF4.stringtochar, but faster and more flexible. + """ + # ensure the array is contiguous + arr = np.array(arr, copy=False, order='C', dtype=np.string_) + return arr.reshape(arr.shape + (1,)).view('S1') + + +def char_to_bytes(arr): + """Convert numpy/dask arrays from characters to fixed width bytes.""" + if arr.dtype != 'S1': + raise ValueError("argument must have dtype='S1'") + + if not arr.ndim: + # no dimension to concatenate along + return arr + + size = arr.shape[-1] + + if not size: + # can't make an S0 dtype + return np.zeros(arr.shape[:-1], dtype=np.string_) + + if isinstance(arr, dask_array_type): + import dask.array as da + + if len(arr.chunks[-1]) > 1: + raise ValueError('cannot stacked dask character array with ' + 'multiple chunks in the last dimension: {}' + .format(arr)) + + dtype = np.dtype('S' + str(arr.shape[-1])) + return da.map_blocks(_numpy_char_to_bytes, arr, + dtype=dtype, + chunks=arr.chunks[:-1], + drop_axis=[arr.ndim - 1]) + else: + return StackedBytesArray(arr) + + +def _numpy_char_to_bytes(arr): + """Like netCDF4.chartostring, but faster and more flexible. + """ + # based on: http://stackoverflow.com/a/10984878/809705 + arr = np.array(arr, copy=False, order='C') + dtype = 'S' + str(arr.shape[-1]) + return arr.view(dtype).reshape(arr.shape[:-1]) + + +class StackedBytesArray(indexing.ExplicitlyIndexedNDArrayMixin): + """Wrapper around array-like objects to create a new indexable object where + values, when accessed, are automatically stacked along the last dimension. + + >>> StackedBytesArray(np.array(['a', 'b', 'c']))[:] + array('abc', + dtype='|S3') + """ + + def __init__(self, array): + """ + Parameters + ---------- + array : array-like + Original array of values to wrap. + """ + if array.dtype != 'S1': + raise ValueError( + "can only use StackedBytesArray if argument has dtype='S1'") + self.array = indexing.as_indexable(array) + + @property + def dtype(self): + return np.dtype('S' + str(self.array.shape[-1])) + + @property + def shape(self): + return self.array.shape[:-1] + + def __repr__(self): + return ('%s(%r)' % (type(self).__name__, self.array)) + + def __getitem__(self, key): + # require slicing the last dimension completely + key = type(key)(indexing.expanded_indexer(key.tuple, self.array.ndim)) + if key.tuple[-1] != slice(None): + raise IndexError('too many indices') + return _numpy_char_to_bytes(self.array[key]) diff --git a/xarray/coding/variables.py b/xarray/coding/variables.py index 4e61e7d4722..1207f5743cb 100644 --- a/xarray/coding/variables.py +++ b/xarray/coding/variables.py @@ -129,10 +129,10 @@ def _apply_mask(data, # type: np.ndarray dtype, # type: Any ): # type: np.ndarray """Mask all matching values in a NumPy arrays.""" + data = np.asarray(data, dtype=dtype) condition = False for fv in encoded_fill_values: condition |= data == fv - data = np.asarray(data, dtype=dtype) return np.where(condition, decoded_fill_value, data) diff --git a/xarray/conventions.py b/xarray/conventions.py index 04664435732..ed90c34387b 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -6,100 +6,15 @@ import numpy as np import pandas as pd -from .coding import times, variables +from .coding import times, strings, variables from .coding.variables import SerializationWarning from .core import duck_array_ops, indexing -from .core.pycompat import OrderedDict, basestring, iteritems, dask_array_type +from .core.pycompat import ( + OrderedDict, basestring, bytes_type, iteritems, dask_array_type, + unicode_type) from .core.variable import IndexVariable, Variable, as_variable -class StackedBytesArray(indexing.ExplicitlyIndexedNDArrayMixin): - """Wrapper around array-like objects to create a new indexable object where - values, when accessed, are automatically stacked along the last dimension. - - >>> StackedBytesArray(np.array(['a', 'b', 'c']))[:] - array('abc', - dtype='|S3') - """ - - def __init__(self, array): - """ - Parameters - ---------- - array : array-like - Original array of values to wrap. - """ - if array.dtype != 'S1': - raise ValueError( - "can only use StackedBytesArray if argument has dtype='S1'") - self.array = indexing.as_indexable(array) - - @property - def dtype(self): - return np.dtype('S' + str(self.array.shape[-1])) - - @property - def shape(self): - return self.array.shape[:-1] - - def __str__(self): - # TODO(shoyer): figure out why we need this special case? - if self.ndim == 0: - return str(np.array(self).item()) - else: - return repr(self) - - def __repr__(self): - return ('%s(%r)' % (type(self).__name__, self.array)) - - def __getitem__(self, key): - # require slicing the last dimension completely - key = type(key)(indexing.expanded_indexer(key.tuple, self.array.ndim)) - if key.tuple[-1] != slice(None): - raise IndexError('too many indices') - return char_to_bytes(self.array[key]) - - -class BytesToStringArray(indexing.ExplicitlyIndexedNDArrayMixin): - """Wrapper that decodes bytes to unicode when values are read. - - >>> BytesToStringArray(np.array([b'abc']))[:] - array(['abc'], - dtype=object) - """ - - def __init__(self, array, encoding='utf-8'): - """ - Parameters - ---------- - array : array-like - Original array of values to wrap. - encoding : str - String encoding to use. - """ - self.array = indexing.as_indexable(array) - self.encoding = encoding - - @property - def dtype(self): - # variable length string - return np.dtype(object) - - def __str__(self): - # TODO(shoyer): figure out why we need this special case? - if self.ndim == 0: - return str(np.array(self).item()) - else: - return repr(self) - - def __repr__(self): - return ('%s(%r, encoding=%r)' - % (type(self).__name__, self.array, self.encoding)) - - def __getitem__(self, key): - return decode_bytes_array(self.array[key], self.encoding) - - class NativeEndiannessArray(indexing.ExplicitlyIndexedNDArrayMixin): """Decode arrays on the fly from non-native to native endianness @@ -159,112 +74,10 @@ def __getitem__(self, key): return np.asarray(self.array[key], dtype=self.dtype) -def bytes_to_char(arr): - """Like netCDF4.stringtochar, but faster and more flexible. - """ - # ensure the array is contiguous - arr = np.array(arr, copy=False, order='C') - kind = arr.dtype.kind - if kind not in ['U', 'S']: - raise ValueError('argument must be a string array') - return arr.reshape(arr.shape + (1,)).view(kind + '1') - - -def char_to_bytes(arr): - """Like netCDF4.chartostring, but faster and more flexible. - """ - # based on: http://stackoverflow.com/a/10984878/809705 - arr = np.array(arr, copy=False, order='C') - - kind = arr.dtype.kind - if kind not in ['U', 'S']: - raise ValueError('argument must be a string array') - - if not arr.ndim: - # no dimension to concatenate along - return arr - - size = arr.shape[-1] - if not size: - # can't make an S0 dtype - return np.zeros(arr.shape[:-1], dtype=kind) - - dtype = kind + str(size) - return arr.view(dtype).reshape(arr.shape[:-1]) - - -def decode_bytes_array(bytes_array, encoding='utf-8'): - # This is faster than using np.char.decode() or np.vectorize() - bytes_array = np.asarray(bytes_array) - decoded = [x.decode(encoding) for x in bytes_array.ravel()] - return np.array(decoded, dtype=object).reshape(bytes_array.shape) - - -def encode_string_array(string_array, encoding='utf-8'): - string_array = np.asarray(string_array) - encoded = [x.encode(encoding) for x in string_array.ravel()] - return np.array(encoded, dtype=bytes).reshape(string_array.shape) - - -def safe_setitem(dest, key, value, name=None): - if key in dest: - var_str = ' on variable {!r}'.format(name) if name else '' - raise ValueError( - 'failed to prevent overwriting existing key {} in attrs{}. ' - 'This is probably an encoding field used by xarray to describe ' - 'how a variable is serialized. To proceed, remove this key from ' - "the variable's attributes manually.".format(key, var_str)) - dest[key] = value - - -def pop_to(source, dest, key, name=None): - """ - A convenience function which pops a key k from source to dest. - None values are not passed on. If k already exists in dest an - error is raised. - """ - value = source.pop(key, None) - if value is not None: - safe_setitem(dest, key, value, name=name) - return value - - def _var_as_tuple(var): return var.dims, var.data, var.attrs.copy(), var.encoding.copy() -def maybe_encode_as_char_array(var, name=None): - if var.dtype.kind in {'S', 'U'}: - dims, data, attrs, encoding = _var_as_tuple(var) - if data.dtype.kind == 'U': - if '_FillValue' in attrs: - raise NotImplementedError( - 'variable {!r} has a _FillValue specified, but ' - '_FillValue is yet supported on unicode strings: ' - 'https://github.com/pydata/xarray/issues/1647' - .format(name)) - - string_encoding = encoding.pop('_Encoding', 'utf-8') - safe_setitem(attrs, '_Encoding', string_encoding, name=name) - data = encode_string_array(data, string_encoding) - - if data.dtype.itemsize > 1: - data = bytes_to_char(data) - dims = dims + ('string%s' % data.shape[-1],) - - var = Variable(dims, data, attrs, encoding) - return var - - -def maybe_encode_string_dtype(var, name=None): - # need to apply after ensure_dtype_not_object() - if 'dtype' in var.encoding and var.encoding['dtype'] == 'S1': - assert var.dtype.kind in {'S', 'U'} - var = maybe_encode_as_char_array(var, name=name) - del var.encoding['dtype'] - return var - - def maybe_encode_nonstring_dtype(var, name=None): if 'dtype' in var.encoding and var.encoding['dtype'] != 'S1': dims, data, attrs, encoding = _var_as_tuple(var) @@ -306,19 +119,23 @@ def _infer_dtype(array, name=None): """Given an object array with no missing values, infer its dtype from its first element """ + if array.dtype.kind != 'O': + raise TypeError('infer_type must be called on a dtype=object array') + if array.size == 0: - dtype = np.dtype(float) - else: - dtype = np.array(array[(0,) * array.ndim]).dtype - if dtype.kind in ['S', 'U']: - # don't just use inferred dtype to avoid truncating arrays to - # the length of their first element - dtype = np.dtype(dtype.kind) - elif dtype.kind == 'O': - raise ValueError('unable to infer dtype on variable {!r}; xarray ' - 'cannot serialize arbitrary Python objects' - .format(name)) - return dtype + return np.dtype(float) + + element = array[(0,) * array.ndim] + if isinstance(element, (bytes_type, unicode_type)): + return strings.create_vlen_dtype(type(element)) + + dtype = np.array(element).dtype + if dtype.kind != 'O': + return dtype + + raise ValueError('unable to infer dtype on variable {!r}; xarray ' + 'cannot serialize arbitrary Python objects' + .format(name)) def ensure_not_multiindex(var, name=None): @@ -332,10 +149,32 @@ def ensure_not_multiindex(var, name=None): 'variables instead.'.format(name)) +def _copy_with_dtype(data, dtype): + """Create a copy of an array with the given dtype. + + We use this instead of np.array() to ensure that custom object dtypes end + up on the resulting array. + """ + result = np.empty(data.shape, dtype) + result[...] = data + return result + + def ensure_dtype_not_object(var, name=None): # TODO: move this from conventions to backends? (it's not CF related) if var.dtype.kind == 'O': dims, data, attrs, encoding = _var_as_tuple(var) + + if isinstance(data, dask_array_type): + warnings.warn( + 'variable {} has data in the form of a dask array with ' + 'dtype=object, which means it is being loaded into memory ' + 'to determine a data type that can be safely stored on disk. ' + 'To avoid this, coerce this variable to a fixed-size dtype ' + 'with astype() before saving it.'.format(name), + SerializationWarning) + data = data.compute() + missing = pd.isnull(data) if missing.any(): # nb. this will fail for dask.array data @@ -345,9 +184,9 @@ def ensure_dtype_not_object(var, name=None): # There is no safe bit-pattern for NA in typical binary string # formats, we so can't set a fill_value. Unfortunately, this means # we can't distinguish between missing values and empty strings. - if inferred_dtype.kind == 'S': + if strings.is_bytes_dtype(inferred_dtype): fill_value = b'' - elif inferred_dtype.kind == 'U': + elif strings.is_unicode_dtype(inferred_dtype): fill_value = u'' else: # insist on using float for numeric values @@ -355,10 +194,12 @@ def ensure_dtype_not_object(var, name=None): inferred_dtype = np.dtype(float) fill_value = inferred_dtype.type(np.nan) - data = np.array(data, dtype=inferred_dtype, copy=True) + data = _copy_with_dtype(data, dtype=inferred_dtype) data[missing] = fill_value else: - data = data.astype(dtype=_infer_dtype(data, name)) + data = _copy_with_dtype(data, dtype=_infer_dtype(data, name)) + + assert data.dtype.kind != 'O' or data.dtype.metadata var = Variable(dims, data, attrs, encoding) return var @@ -397,7 +238,6 @@ def encode_cf_variable(var, needs_copy=True, name=None): var = maybe_default_fill_value(var) var = maybe_encode_bools(var) var = ensure_dtype_not_object(var, name=name) - var = maybe_encode_string_dtype(var, name=name) return var @@ -439,32 +279,20 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True, out : Variable A variable holding the decoded equivalent of var. """ - # use _data instead of data so as not to trigger loading data var = as_variable(var) - data = var._data - dimensions = var.dims - attributes = var.attrs.copy() - encoding = var.encoding.copy() - - original_dtype = data.dtype + original_dtype = var.dtype - if concat_characters and data.dtype.kind == 'S': + if concat_characters: if stack_char_dim: - dimensions = dimensions[:-1] - data = StackedBytesArray(data) - - string_encoding = pop_to(attributes, encoding, '_Encoding') - if string_encoding is not None: - data = BytesToStringArray(data, string_encoding) - - # TODO(shoyer): convert everything above to use coders - var = Variable(dimensions, data, attributes, encoding) + var = strings.CharacterArrayCoder().decode(var, name=name) + var = strings.EncodedStringCoder().decode(var) if mask_and_scale: for coder in [variables.UnsignedIntegerCoder(), variables.CFMaskCoder(), variables.CFScaleOffsetCoder()]: var = coder.decode(var, name=name) + if decode_times: for coder in [times.CFTimedeltaCoder(), times.CFDatetimeCoder()]: @@ -492,6 +320,7 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True, if not isinstance(data, dask_array_type): data = indexing.LazilyOuterIndexedArray(data) + return Variable(dimensions, data, attributes, encoding=encoding) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 1e69363225f..7f8a440ba5d 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1371,24 +1371,6 @@ def test_group(self): open_kwargs={'group': group}) as actual: assert_identical(original, actual) - # TODO: implement zarr object encoding and make these tests pass - @pytest.mark.xfail(reason="Zarr object encoding not implemented") - def test_multiindex_not_implemented(self): - super(CFEncodedDataTest, self).test_multiindex_not_implemented() - - @pytest.mark.xfail(reason="Zarr object encoding not implemented") - def test_roundtrip_bytes_with_fill_value(self): - super(CFEncodedDataTest, self).test_roundtrip_bytes_with_fill_value() - - @pytest.mark.xfail(reason="Zarr object encoding not implemented") - def test_roundtrip_object_dtype(self): - super(CFEncodedDataTest, self).test_roundtrip_object_dtype() - - @pytest.mark.xfail(reason="Zarr object encoding not implemented") - def test_roundtrip_string_encoded_characters(self): - super(CFEncodedDataTest, - self).test_roundtrip_string_encoded_characters() - # TODO: someone who understand caching figure out whether chaching # makes sense for Zarr backend @pytest.mark.xfail(reason="Zarr caching not implemented") diff --git a/xarray/tests/test_coding_strings.py b/xarray/tests/test_coding_strings.py new file mode 100644 index 00000000000..53d028e164b --- /dev/null +++ b/xarray/tests/test_coding_strings.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function + +import numpy as np +import pytest + +from xarray import Variable +from xarray.core.pycompat import bytes_type, unicode_type, suppress +from xarray.coding import strings +from xarray.core import indexing + +from . import (IndexerMaker, assert_array_equal, assert_identical, + raises_regex, requires_dask) + + +with suppress(ImportError): + import dask.array as da + + +def test_vlen_dtype(): + dtype = strings.create_vlen_dtype(unicode_type) + assert dtype.metadata['element_type'] == unicode_type + assert strings.is_unicode_dtype(dtype) + assert not strings.is_bytes_dtype(dtype) + assert strings.check_vlen_dtype(dtype) is unicode_type + + dtype = strings.create_vlen_dtype(bytes_type) + assert dtype.metadata['element_type'] == bytes_type + assert not strings.is_unicode_dtype(dtype) + assert strings.is_bytes_dtype(dtype) + assert strings.check_vlen_dtype(dtype) is bytes_type + + assert strings.check_vlen_dtype(np.dtype(object)) is None + + +def test_EncodedStringCoder_decode(): + coder = strings.EncodedStringCoder() + + raw_data = np.array([b'abc', u'ß∂µ∆'.encode('utf-8')]) + raw = Variable(('x',), raw_data, {'_Encoding': 'utf-8'}) + actual = coder.decode(raw) + + expected = Variable( + ('x',), np.array([u'abc', u'ß∂µ∆'], dtype=object)) + assert_identical(actual, expected) + + assert_identical(coder.decode(actual[0]), expected[0]) + + +@requires_dask +def test_EncodedStringCoder_decode_dask(): + coder = strings.EncodedStringCoder() + + raw_data = np.array([b'abc', u'ß∂µ∆'.encode('utf-8')]) + raw = Variable(('x',), raw_data, {'_Encoding': 'utf-8'}).chunk() + actual = coder.decode(raw) + assert isinstance(actual.data, da.Array) + + expected = Variable(('x',), np.array([u'abc', u'ß∂µ∆'], dtype=object)) + assert_identical(actual, expected) + + actual_indexed = coder.decode(actual[0]) + assert isinstance(actual_indexed.data, da.Array) + assert_identical(actual_indexed, expected[0]) + + +def test_EncodedStringCoder_encode(): + dtype = strings.create_vlen_dtype(unicode_type) + raw_data = np.array([u'abc', u'ß∂µ∆'], dtype=dtype) + expected_data = np.array([r.encode('utf-8') for r in raw_data], + dtype=object) + + coder = strings.EncodedStringCoder(allows_unicode=True) + raw = Variable(('x',), raw_data, encoding={'dtype': 'S1'}) + actual = coder.encode(raw) + expected = Variable(('x',), expected_data, attrs={'_Encoding': 'utf-8'}) + assert_identical(actual, expected) + + raw = Variable(('x',), raw_data) + assert_identical(coder.encode(raw), raw) + + coder = strings.EncodedStringCoder(allows_unicode=False) + assert_identical(coder.encode(raw), expected) + + +@pytest.mark.parametrize('original', [ + Variable(('x',), [b'ab', b'cdef']), + Variable((), b'ab'), + Variable(('x',), [b'a', b'b']), + Variable((), b'a'), +]) +def test_CharacterArrayCoder_roundtrip(original): + coder = strings.CharacterArrayCoder() + roundtripped = coder.decode(coder.encode(original)) + assert_identical(original, roundtripped) + + +@pytest.mark.parametrize('data', [ + np.array([b'a', b'bc']), + np.array([b'a', b'bc'], dtype=strings.create_vlen_dtype(bytes_type)), +]) +def test_CharacterArrayCoder_encode(data): + coder = strings.CharacterArrayCoder() + raw = Variable(('x',), data) + actual = coder.encode(raw) + expected = Variable(('x', 'string2'), + np.array([[b'a', b''], [b'b', b'c']])) + assert_identical(actual, expected) + + +def test_StackedBytesArray(): + array = np.array([[b'a', b'b', b'c'], [b'd', b'e', b'f']], dtype='S') + actual = strings.StackedBytesArray(array) + expected = np.array([b'abc', b'def'], dtype='S') + assert actual.dtype == expected.dtype + assert actual.shape == expected.shape + assert actual.size == expected.size + assert actual.ndim == expected.ndim + assert len(actual) == len(expected) + assert_array_equal(expected, actual) + + B = IndexerMaker(indexing.BasicIndexer) + assert_array_equal(expected[:1], actual[B[:1]]) + with pytest.raises(IndexError): + actual[B[:, :2]] + + +def test_StackedBytesArray_scalar(): + array = np.array([b'a', b'b', b'c'], dtype='S') + actual = strings.StackedBytesArray(array) + + expected = np.array(b'abc') + assert actual.dtype == expected.dtype + assert actual.shape == expected.shape + assert actual.size == expected.size + assert actual.ndim == expected.ndim + with pytest.raises(TypeError): + len(actual) + np.testing.assert_array_equal(expected, actual) + + B = IndexerMaker(indexing.BasicIndexer) + with pytest.raises(IndexError): + actual[B[:2]] + + +def test_StackedBytesArray_vectorized_indexing(): + array = np.array([[b'a', b'b', b'c'], [b'd', b'e', b'f']], dtype='S') + stacked = strings.StackedBytesArray(array) + expected = np.array([[b'abc', b'def'], [b'def', b'abc']]) + + V = IndexerMaker(indexing.VectorizedIndexer) + indexer = V[np.array([[0, 1], [1, 0]])] + actual = stacked[indexer] + assert_array_equal(actual, expected) + + +def test_char_to_bytes(): + array = np.array([[b'a', b'b', b'c'], [b'd', b'e', b'f']]) + expected = np.array([b'abc', b'def']) + actual = strings.char_to_bytes(array) + assert_array_equal(actual, expected) + + expected = np.array([b'ad', b'be', b'cf']) + actual = strings.char_to_bytes(array.T) # non-contiguous + assert_array_equal(actual, expected) + + +def test_char_to_bytes_ndim_zero(): + expected = np.array(b'a') + actual = strings.char_to_bytes(expected) + assert_array_equal(actual, expected) + + +def test_char_to_bytes_size_zero(): + array = np.zeros((3, 0), dtype='S1') + expected = np.array([b'', b'', b'']) + actual = strings.char_to_bytes(array) + assert_array_equal(actual, expected) + + +@requires_dask +def test_char_to_bytes_dask(): + numpy_array = np.array([[b'a', b'b', b'c'], [b'd', b'e', b'f']]) + array = da.from_array(numpy_array, ((2,), (3,))) + expected = np.array([b'abc', b'def']) + actual = strings.char_to_bytes(array) + assert isinstance(actual, da.Array) + assert actual.chunks == ((2,),) + assert actual.dtype == 'S3' + assert_array_equal(np.array(actual), expected) + + with raises_regex(ValueError, 'stacked dask character array'): + strings.char_to_bytes(array.rechunk(1)) + + +def test_bytes_to_char(): + array = np.array([[b'ab', b'cd'], [b'ef', b'gh']]) + expected = np.array([[[b'a', b'b'], [b'c', b'd']], + [[b'e', b'f'], [b'g', b'h']]]) + actual = strings.bytes_to_char(array) + assert_array_equal(actual, expected) + + expected = np.array([[[b'a', b'b'], [b'e', b'f']], + [[b'c', b'd'], [b'g', b'h']]]) + actual = strings.bytes_to_char(array.T) # non-contiguous + assert_array_equal(actual, expected) + + +@requires_dask +def test_bytes_to_char_dask(): + numpy_array = np.array([b'ab', b'cd']) + array = da.from_array(numpy_array, ((1, 1),)) + expected = np.array([[b'a', b'b'], [b'c', b'd']]) + actual = strings.bytes_to_char(array) + assert isinstance(actual, da.Array) + assert actual.chunks == ((1, 1), ((2,))) + assert actual.dtype == 'S1' + assert_array_equal(np.array(actual), expected) diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 80c69ece5dd..62ff8d7ee1a 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -8,133 +8,18 @@ import pandas as pd import pytest -from xarray import Dataset, Variable, conventions, open_dataset +from xarray import (Dataset, Variable, SerializationWarning, coding, + conventions, open_dataset) from xarray.backends.common import WritableCFDataStore from xarray.backends.memory import InMemoryDataStore from xarray.conventions import decode_cf -from xarray.core import indexing, utils -from xarray.core.pycompat import iteritems from xarray.testing import assert_identical from . import ( - IndexerMaker, TestCase, assert_array_equal, raises_regex, requires_netCDF4, + TestCase, assert_array_equal, raises_regex, requires_netCDF4, requires_cftime_or_netCDF4, unittest, requires_dask) from .test_backends import CFEncodedDataTest -B = IndexerMaker(indexing.BasicIndexer) -V = IndexerMaker(indexing.VectorizedIndexer) - - -class TestStackedBytesArray(TestCase): - def test_wrapper_class(self): - array = np.array([[b'a', b'b', b'c'], [b'd', b'e', b'f']], dtype='S') - actual = conventions.StackedBytesArray(array) - expected = np.array([b'abc', b'def'], dtype='S') - assert actual.dtype == expected.dtype - assert actual.shape == expected.shape - assert actual.size == expected.size - assert actual.ndim == expected.ndim - assert len(actual) == len(expected) - assert_array_equal(expected, actual) - assert_array_equal(expected[:1], actual[B[:1]]) - with pytest.raises(IndexError): - actual[B[:, :2]] - - def test_scalar(self): - array = np.array([b'a', b'b', b'c'], dtype='S') - actual = conventions.StackedBytesArray(array) - - expected = np.array(b'abc') - assert actual.dtype == expected.dtype - assert actual.shape == expected.shape - assert actual.size == expected.size - assert actual.ndim == expected.ndim - with pytest.raises(TypeError): - len(actual) - np.testing.assert_array_equal(expected, actual) - with pytest.raises(IndexError): - actual[B[:2]] - assert str(actual) == str(expected) - - def test_char_to_bytes(self): - array = np.array([['a', 'b', 'c'], ['d', 'e', 'f']]) - expected = np.array(['abc', 'def']) - actual = conventions.char_to_bytes(array) - assert_array_equal(actual, expected) - - expected = np.array(['ad', 'be', 'cf']) - actual = conventions.char_to_bytes(array.T) # non-contiguous - assert_array_equal(actual, expected) - - def test_char_to_bytes_ndim_zero(self): - expected = np.array('a') - actual = conventions.char_to_bytes(expected) - assert_array_equal(actual, expected) - - def test_char_to_bytes_size_zero(self): - array = np.zeros((3, 0), dtype='S1') - expected = np.array([b'', b'', b'']) - actual = conventions.char_to_bytes(array) - assert_array_equal(actual, expected) - - def test_bytes_to_char(self): - array = np.array([['ab', 'cd'], ['ef', 'gh']]) - expected = np.array([[['a', 'b'], ['c', 'd']], - [['e', 'f'], ['g', 'h']]]) - actual = conventions.bytes_to_char(array) - assert_array_equal(actual, expected) - - expected = np.array([[['a', 'b'], ['e', 'f']], - [['c', 'd'], ['g', 'h']]]) - actual = conventions.bytes_to_char(array.T) - assert_array_equal(actual, expected) - - def test_vectorized_indexing(self): - array = np.array([[b'a', b'b', b'c'], [b'd', b'e', b'f']], dtype='S') - stacked = conventions.StackedBytesArray(array) - expected = np.array([[b'abc', b'def'], [b'def', b'abc']]) - indexer = V[np.array([[0, 1], [1, 0]])] - actual = stacked[indexer] - assert_array_equal(actual, expected) - - -class TestBytesToStringArray(TestCase): - - def test_encoding(self): - encoding = 'utf-8' - raw_array = np.array([b'abc', u'ß∂µ∆'.encode(encoding)]) - actual = conventions.BytesToStringArray(raw_array, encoding=encoding) - expected = np.array([u'abc', u'ß∂µ∆'], dtype=object) - - assert actual.dtype == expected.dtype - assert actual.shape == expected.shape - assert actual.size == expected.size - assert actual.ndim == expected.ndim - assert_array_equal(expected, actual) - assert_array_equal(expected[0], actual[B[0]]) - - def test_scalar(self): - expected = np.array(u'abc', dtype=object) - actual = conventions.BytesToStringArray( - np.array(b'abc'), encoding='utf-8') - assert actual.dtype == expected.dtype - assert actual.shape == expected.shape - assert actual.size == expected.size - assert actual.ndim == expected.ndim - with pytest.raises(TypeError): - len(actual) - np.testing.assert_array_equal(expected, actual) - with pytest.raises(IndexError): - actual[B[:2]] - assert str(actual) == str(expected) - - def test_decode_bytes_array(self): - encoding = 'utf-8' - raw_array = np.array([b'abc', u'ß∂µ∆'.encode(encoding)]) - expected = np.array([u'abc', u'ß∂µ∆'], dtype=object) - actual = conventions.decode_bytes_array(raw_array, encoding) - np.testing.assert_array_equal(actual, expected) - class TestBoolTypeArray(TestCase): def test_booltype_array(self): @@ -238,6 +123,15 @@ def test_multidimensional_coordinates(self): # Should not have any global coordinates. assert 'coordinates' not in attrs + @requires_dask + def test_string_object_warning(self): + original = Variable( + ('x',), np.array([u'foo', u'bar'], dtype=object)).chunk() + with pytest.warns(SerializationWarning, + match='dask array with dtype=object'): + encoded = conventions.encode_cf_variable(original) + assert_identical(original, encoded) + @requires_cftime_or_netCDF4 class TestDecodeCF(TestCase): @@ -334,41 +228,28 @@ def test_decode_cf_datetime_transition_to_invalid(self): @requires_dask def test_decode_cf_with_dask(self): - import dask + import dask.array as da original = Dataset({ 't': ('t', [0, 1, 2], {'units': 'days since 2000-01-01'}), 'foo': ('t', [0, 0, 0], {'coordinates': 'y', 'units': 'bar'}), + 'bar': ('string2', [b'a', b'b']), + 'baz': (('x'), [b'abc'], {'_Encoding': 'utf-8'}), 'y': ('t', [5, 10, -999], {'_FillValue': -999}) - }).chunk({'t': 1}) + }).chunk() decoded = conventions.decode_cf(original) - assert dask.is_dask_collection(decoded.y.data) + print(decoded) + assert all(isinstance(var.data, da.Array) + for name, var in decoded.variables.items() + if name not in decoded.indexes) + assert_identical(decoded, conventions.decode_cf(original).compute()) class CFEncodedInMemoryStore(WritableCFDataStore, InMemoryDataStore): - pass - - -class NullWrapper(utils.NDArrayMixin): - """ - Just for testing, this lets us create a numpy array directly - but make it look like its not in memory yet. - """ - - def __init__(self, array): - self.array = array - - def __getitem__(self, key): - return self.array[indexing.orthogonal_indexer(key, self.shape)] - - -def null_wrap(ds): - """ - Given a data store this wraps each variable in a NullWrapper so that - it appears to be out of memory. - """ - variables = dict((k, Variable(v.dims, NullWrapper(v.values), v.attrs)) - for k, v in iteritems(ds)) - return InMemoryDataStore(variables=variables, attributes=ds.attrs) + def encode_variable(self, var): + """encode one variable""" + coder = coding.strings.EncodedStringCoder(allows_unicode=True) + var = coder.encode(var) + return var @requires_netCDF4 From d1e1440dc5d0bc9c341da20fde85b56f2a3c1b5b Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Mon, 30 Apr 2018 19:17:52 +0200 Subject: [PATCH 088/282] DOC: uniformize variable names in indexing.rst (#2091) --- ...t_colorbar.py => plot_control_colorbar.py} | 0 doc/indexing.rst | 118 ++++++++++-------- 2 files changed, 64 insertions(+), 54 deletions(-) rename doc/gallery/{control_plot_colorbar.py => plot_control_colorbar.py} (100%) diff --git a/doc/gallery/control_plot_colorbar.py b/doc/gallery/plot_control_colorbar.py similarity index 100% rename from doc/gallery/control_plot_colorbar.py rename to doc/gallery/plot_control_colorbar.py diff --git a/doc/indexing.rst b/doc/indexing.rst index 6bdffde23c2..1f6ae006cf7 100644 --- a/doc/indexing.rst +++ b/doc/indexing.rst @@ -35,15 +35,15 @@ below and summarized in this table: +------------------+--------------+---------------------------------+--------------------------------+ | Dimension lookup | Index lookup | ``DataArray`` syntax | ``Dataset`` syntax | +==================+==============+=================================+================================+ -| Positional | By integer | ``arr[:, 0]`` | *not available* | +| Positional | By integer | ``da[:, 0]`` | *not available* | +------------------+--------------+---------------------------------+--------------------------------+ -| Positional | By label | ``arr.loc[:, 'IA']`` | *not available* | +| Positional | By label | ``da.loc[:, 'IA']`` | *not available* | +------------------+--------------+---------------------------------+--------------------------------+ -| By name | By integer | ``arr.isel(space=0)`` or |br| | ``ds.isel(space=0)`` or |br| | -| | | ``arr[dict(space=0)]`` | ``ds[dict(space=0)]`` | +| By name | By integer | ``da.isel(space=0)`` or |br| | ``ds.isel(space=0)`` or |br| | +| | | ``da[dict(space=0)]`` | ``ds[dict(space=0)]`` | +------------------+--------------+---------------------------------+--------------------------------+ -| By name | By label | ``arr.sel(space='IA')`` or |br| | ``ds.sel(space='IA')`` or |br| | -| | | ``arr.loc[dict(space='IA')]`` | ``ds.loc[dict(space='IA')]`` | +| By name | By label | ``da.sel(space='IA')`` or |br| | ``ds.sel(space='IA')`` or |br| | +| | | ``da.loc[dict(space='IA')]`` | ``ds.loc[dict(space='IA')]`` | +------------------+--------------+---------------------------------+--------------------------------+ More advanced indexing is also possible for all the methods by @@ -60,19 +60,19 @@ DataArray: .. ipython:: python - arr = xr.DataArray(np.random.rand(4, 3), - [('time', pd.date_range('2000-01-01', periods=4)), - ('space', ['IA', 'IL', 'IN'])]) - arr[:2] - arr[0, 0] - arr[:, [2, 1]] + da = xr.DataArray(np.random.rand(4, 3), + [('time', pd.date_range('2000-01-01', periods=4)), + ('space', ['IA', 'IL', 'IN'])]) + da[:2] + da[0, 0] + da[:, [2, 1]] Attributes are persisted in all indexing operations. .. warning:: Positional indexing deviates from the NumPy when indexing with multiple - arrays like ``arr[[0, 1], [0, 1]]``, as described in + arrays like ``da[[0, 1], [0, 1]]``, as described in :ref:`vectorized_indexing`. xarray also supports label-based indexing, just like pandas. Because @@ -81,7 +81,7 @@ fast. To do label based indexing, use the :py:attr:`~xarray.DataArray.loc` attri .. ipython:: python - arr.loc['2000-01-01':'2000-01-02', 'IA'] + da.loc['2000-01-01':'2000-01-02', 'IA'] In this example, the selected is a subpart of the array in the range '2000-01-01':'2000-01-02' along the first coordinate `time` @@ -98,8 +98,8 @@ Setting values with label based indexing is also supported: .. ipython:: python - arr.loc['2000-01-01', ['IL', 'IN']] = -10 - arr + da.loc['2000-01-01', ['IL', 'IN']] = -10 + da Indexing with dimension names @@ -114,10 +114,10 @@ use them explicitly to slice data. There are two ways to do this: .. ipython:: python # index by integer array indices - arr[dict(space=0, time=slice(None, 2))] + da[dict(space=0, time=slice(None, 2))] # index by dimension coordinate labels - arr.loc[dict(time=slice('2000-01-01', '2000-01-02'))] + da.loc[dict(time=slice('2000-01-01', '2000-01-02'))] 2. Use the :py:meth:`~xarray.DataArray.sel` and :py:meth:`~xarray.DataArray.isel` convenience methods: @@ -125,10 +125,10 @@ use them explicitly to slice data. There are two ways to do this: .. ipython:: python # index by integer array indices - arr.isel(space=0, time=slice(None, 2)) + da.isel(space=0, time=slice(None, 2)) # index by dimension coordinate labels - arr.sel(time=slice('2000-01-01', '2000-01-02')) + da.sel(time=slice('2000-01-01', '2000-01-02')) The arguments to these methods can be any objects that could index the array along the dimension given by the keyword, e.g., labels for an individual value, @@ -138,7 +138,7 @@ Python :py:func:`slice` objects or 1-dimensional arrays. We would love to be able to do indexing with labeled dimension names inside brackets, but unfortunately, Python `does yet not support`__ indexing with - keyword arguments like ``arr[space=0]`` + keyword arguments like ``da[space=0]`` __ http://legacy.python.org/dev/peps/pep-0472/ @@ -156,16 +156,16 @@ enabling nearest neighbor (inexact) lookups by use of the methods ``'pad'``, .. ipython:: python - data = xr.DataArray([1, 2, 3], [('x', [0, 1, 2])]) - data.sel(x=[1.1, 1.9], method='nearest') - data.sel(x=0.1, method='backfill') - data.reindex(x=[0.5, 1, 1.5, 2, 2.5], method='pad') + da = xr.DataArray([1, 2, 3], [('x', [0, 1, 2])]) + da.sel(x=[1.1, 1.9], method='nearest') + da.sel(x=0.1, method='backfill') + da.reindex(x=[0.5, 1, 1.5, 2, 2.5], method='pad') Tolerance limits the maximum distance for valid matches with an inexact lookup: .. ipython:: python - data.reindex(x=[1.1, 1.5], method='nearest', tolerance=0.2) + da.reindex(x=[1.1, 1.5], method='nearest', tolerance=0.2) The method parameter is not yet supported if any of the arguments to ``.sel()`` is a ``slice`` object: @@ -173,7 +173,7 @@ to ``.sel()`` is a ``slice`` object: .. ipython:: :verbatim: - In [1]: data.sel(x=slice(1, 3), method='nearest') + In [1]: da.sel(x=slice(1, 3), method='nearest') NotImplementedError However, you don't need to use ``method`` to do inexact slicing. Slicing @@ -182,15 +182,15 @@ labels are monotonic increasing: .. ipython:: python - data.sel(x=slice(0.9, 3.1)) + da.sel(x=slice(0.9, 3.1)) Indexing axes with monotonic decreasing labels also works, as long as the ``slice`` or ``.loc`` arguments are also decreasing: .. ipython:: python - reversed_data = data[::-1] - reversed_data.loc[3.1:0.9] + reversed_da = da[::-1] + reversed_da.loc[3.1:0.9] Dataset indexing @@ -201,7 +201,10 @@ simultaneously, returning a new dataset: .. ipython:: python - ds = arr.to_dataset(name='foo') + da = xr.DataArray(np.random.rand(4, 3), + [('time', pd.date_range('2000-01-01', periods=4)), + ('space', ['IA', 'IL', 'IN'])]) + ds = da.to_dataset(name='foo') ds.isel(space=[0], time=[0]) ds.sel(time='2000-01-01') @@ -243,8 +246,8 @@ xarray, use :py:meth:`~xarray.DataArray.where`: .. ipython:: python - arr2 = xr.DataArray(np.arange(16).reshape(4, 4), dims=['x', 'y']) - arr2.where(arr2.x + arr2.y < 4) + da = xr.DataArray(np.arange(16).reshape(4, 4), dims=['x', 'y']) + da.where(da.x + da.y < 4) This is particularly useful for ragged indexing of multi-dimensional data, e.g., to apply a 2D mask to an image. Note that ``where`` follows all the @@ -254,7 +257,7 @@ usual xarray broadcasting and alignment rules for binary operations (e.g., .. ipython:: python - arr2.where(arr2.y < 2) + da.where(da.y < 2) By default ``where`` maintains the original size of the data. For cases where the selected data size is much smaller than the original data, @@ -263,7 +266,7 @@ elements that are fully masked: .. ipython:: python - arr2.where(arr2.y < 2, drop=True) + da.where(da.y < 2, drop=True) .. _selecting values with isin: @@ -276,8 +279,8 @@ multiple values, use :py:meth:`~xarray.DataArray.isin`: .. ipython:: python - arr = xr.DataArray([1, 2, 3, 4, 5], dims=['x']) - arr.isin([2, 4]) + da = xr.DataArray([1, 2, 3, 4, 5], dims=['x']) + da.isin([2, 4]) :py:meth:`~xarray.DataArray.isin` works particularly well with :py:meth:`~xarray.DataArray.where` to support indexing by arrays that are not @@ -286,7 +289,7 @@ already labels of an array: .. ipython:: python lookup = xr.DataArray([-1, -2, -3, -4, -5], dims=['x']) - arr.where(lookup.isin([-2, -4]), drop=True) + da.where(lookup.isin([-2, -4]), drop=True) However, some caution is in order: when done repeatedly, this type of indexing is significantly slower than using :py:meth:`~xarray.DataArray.sel`. @@ -364,8 +367,8 @@ These methods may and also be applied to ``Dataset`` objects .. ipython:: python - ds2 = da.to_dataset(name='bar') - ds2.isel(x=xr.DataArray([0, 1, 2], dims=['points'])) + ds = da.to_dataset(name='bar') + ds.isel(x=xr.DataArray([0, 1, 2], dims=['points'])) .. tip:: @@ -446,7 +449,7 @@ __ https://docs.scipy.org/doc/numpy/user/basics.indexing.html#assigning-values-t or ``sel``:: # DO NOT do this - arr.isel(space=0) = 0 + da.isel(space=0) = 0 Assigning values with the chained indexing using ``.sel`` or ``.isel`` fails silently. @@ -490,10 +493,13 @@ method: .. ipython:: python + da = xr.DataArray(np.random.rand(4, 3), + [('time', pd.date_range('2000-01-01', periods=4)), + ('space', ['IA', 'IL', 'IN'])]) times = xr.DataArray(pd.to_datetime(['2000-01-03', '2000-01-02', '2000-01-01']), dims='new_time') - arr.sel(space=xr.DataArray(['IA', 'IL', 'IN'], dims=['new_time']), - time=times) + da.sel(space=xr.DataArray(['IA', 'IL', 'IN'], dims=['new_time']), + time=times) .. _align and reindex: @@ -515,15 +521,15 @@ To reindex a particular dimension, use :py:meth:`~xarray.DataArray.reindex`: .. ipython:: python - arr.reindex(space=['IA', 'CA']) + da.reindex(space=['IA', 'CA']) The :py:meth:`~xarray.DataArray.reindex_like` method is a useful shortcut. To demonstrate, we will make a subset DataArray with new values: .. ipython:: python - foo = arr.rename('foo') - baz = (10 * arr[:2, :2]).rename('baz') + foo = da.rename('foo') + baz = (10 * da[:2, :2]).rename('baz') baz Reindexing ``foo`` with ``baz`` selects out the first two values along each @@ -570,8 +576,8 @@ integer-based indexing as a fallback for dimensions without a coordinate label: .. ipython:: python - array = xr.DataArray([1, 2, 3], dims='x') - array.sel(x=[0, -1]) + da = xr.DataArray([1, 2, 3], dims='x') + da.sel(x=[0, -1]) Alignment between xarray objects where one or both do not have coordinate labels succeeds only if all dimensions of the same name have the same length. @@ -580,7 +586,7 @@ Otherwise, it raises an informative error: .. ipython:: :verbatim: - In [62]: xr.align(array, array[:2]) + In [62]: xr.align(da, da[:2]) ValueError: arguments without labels along dimension 'x' cannot be aligned because they have different dimension sizes: {2, 3} Underlying Indexes @@ -592,9 +598,12 @@ through the :py:attr:`~xarray.DataArray.indexes` attribute. .. ipython:: python - arr - arr.indexes - arr.indexes['time'] + da = xr.DataArray(np.random.rand(4, 3), + [('time', pd.date_range('2000-01-01', periods=4)), + ('space', ['IA', 'IL', 'IN'])]) + da + da.indexes + da.indexes['time'] Use :py:meth:`~xarray.DataArray.get_index` to get an index for a dimension, falling back to a default :py:class:`pandas.RangeIndex` if it has no coordinate @@ -602,8 +611,9 @@ labels: .. ipython:: python - array - array.get_index('x') + da = xr.DataArray([1, 2, 3], dims='x') + da + da.get_index('x') .. _copies_vs_views: From 39b2a37207fc8e6c5199ba9386831ba7eb06d82b Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Tue, 1 May 2018 16:23:59 +0900 Subject: [PATCH 089/282] Remove flake8 from travis (#1919) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5a9bea81e4c..e375f6fb063 100644 --- a/.travis.yml +++ b/.travis.yml @@ -101,7 +101,6 @@ install: - python xarray/util/print_versions.py script: - - flake8 -j auto xarray - python -OO -c "import xarray" - if [[ "$CONDA_ENV" == "docs" ]]; then conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc; From 39bd2076e87090ef3130f55f472f3138abad3558 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 1 May 2018 17:54:44 -0700 Subject: [PATCH 090/282] Install cftime with pip on Windows (#2098) --- ci/requirements-py27-windows.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/requirements-py27-windows.yml b/ci/requirements-py27-windows.yml index 43b292100de..7562874785b 100644 --- a/ci/requirements-py27-windows.yml +++ b/ci/requirements-py27-windows.yml @@ -3,7 +3,6 @@ channels: - conda-forge dependencies: - python=2.7 - - cftime - dask - distributed - h5py @@ -21,3 +20,5 @@ dependencies: - toolz - rasterio - zarr + - pip: + - cftime From 0cc64a08c672e6361d05acea3fea9f34308b62ed Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Wed, 2 May 2018 11:31:01 +0900 Subject: [PATCH 091/282] Drop conflicted coordinate when assignment. (#2087) * Drop conflicted coordinate when assignment. * flake8 * Python 2 support. Avoid overwrite variables to be assigned. * More pythonic --- doc/whats-new.rst | 4 ++++ xarray/core/merge.py | 17 +++++++++++++++- xarray/tests/test_dataset.py | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d804fe7b915..59aad702eea 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -40,6 +40,10 @@ Enhancements Bug fixes ~~~~~~~~~ +- When assigning a :py:class:`DataArray` to :py:class:`Dataset`, any conflicted + non-dimensional coordinates of the DataArray are now dropped. + (:issue:`2068`) + By `Keisuke Fujii `_. - Better error handling in ``open_mfdataset`` (:issue:`2077`). By `Stephan Hoyer `_. diff --git a/xarray/core/merge.py b/xarray/core/merge.py index 7069ca9d96b..8ecb2c338b4 100644 --- a/xarray/core/merge.py +++ b/xarray/core/merge.py @@ -547,6 +547,21 @@ def dataset_merge_method(dataset, other, overwrite_vars, compat, join): def dataset_update_method(dataset, other): - """Guts of the Dataset.update method""" + """Guts of the Dataset.update method + + This drops a duplicated coordinates from `other` (GH:2068) + """ + from .dataset import Dataset + from .dataarray import DataArray + + other = other.copy() + for k, obj in other.items(): + if isinstance(obj, (Dataset, DataArray)): + # drop duplicated coordinates + coord_names = [c for c in obj.coords + if c not in obj.dims and c in dataset.coords] + if coord_names: + other[k] = obj.drop(*coord_names) + return merge_core([dataset, other], priority_arg=1, indexes=dataset.indexes) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index cbd1cd33f2c..51871472983 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2327,6 +2327,45 @@ def test_setitem_auto_align(self): expected = Dataset({'x': ('y', [4, 5, 6])}, {'y': range(3)}) assert_identical(ds, expected) + def test_setitem_with_coords(self): + # Regression test for GH:2068 + ds = create_test_data() + + other = DataArray(np.arange(10), dims='dim3', + coords={'numbers': ('dim3', np.arange(10))}) + expected = ds.copy() + expected['var3'] = other.drop('numbers') + actual = ds.copy() + actual['var3'] = other + assert_identical(expected, actual) + assert 'numbers' in other # should not change other + + # with alignment + other = ds['var3'].isel(dim3=slice(1, -1)) + other['numbers'] = ('dim3', np.arange(8)) + actual = ds.copy() + actual['var3'] = other + assert 'numbers' in other # should not change other + expected = ds.copy() + expected['var3'] = ds['var3'].isel(dim3=slice(1, -1)) + assert_identical(expected, actual) + + # with non-duplicate coords + other = ds['var3'].isel(dim3=slice(1, -1)) + other['numbers'] = ('dim3', np.arange(8)) + other['position'] = ('dim3', np.arange(8)) + actual = ds.copy() + actual['var3'] = other + assert 'position' in actual + assert 'position' in other + + # assigning a coordinate-only dataarray + actual = ds.copy() + other = actual['numbers'] + other[0] = 10 + actual['numbers'] = other + assert actual['numbers'][0] == 10 + def test_setitem_align_new_indexes(self): ds = Dataset({'foo': ('x', [1, 2, 3])}, {'x': [0, 1, 2]}) ds['bar'] = DataArray([2, 3, 4], [('x', [1, 2, 3])]) From b9f40cc1da9c45b3dd33a3434b69c3d8fce57138 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Thu, 3 May 2018 06:59:33 +0900 Subject: [PATCH 092/282] Fix a bug introduced in #2087 (#2100) * Bugfix introduced by #2087 * flake8 * Remove trailing whitespaces --- xarray/core/merge.py | 2 +- xarray/tests/test_dataset.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/xarray/core/merge.py b/xarray/core/merge.py index 8ecb2c338b4..d3c9871abef 100644 --- a/xarray/core/merge.py +++ b/xarray/core/merge.py @@ -561,7 +561,7 @@ def dataset_update_method(dataset, other): coord_names = [c for c in obj.coords if c not in obj.dims and c in dataset.coords] if coord_names: - other[k] = obj.drop(*coord_names) + other[k] = obj.drop(coord_names) return merge_core([dataset, other], priority_arg=1, indexes=dataset.indexes) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 51871472983..b99f7ea1eec 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2366,6 +2366,13 @@ def test_setitem_with_coords(self): actual['numbers'] = other assert actual['numbers'][0] == 10 + # GH: 2099 + ds = Dataset({'var': ('x', [1, 2, 3])}, + coords={'x': [0, 1, 2], 'z1': ('x', [1, 2, 3]), + 'z2': ('x', [1, 2, 3])}) + ds['var'] = ds['var'] * 2 + assert np.allclose(ds['var'], [2, 4, 6]) + def test_setitem_align_new_indexes(self): ds = Dataset({'foo': ('x', [1, 2, 3])}, {'x': [0, 1, 2]}) ds['bar'] = DataArray([2, 3, 4], [('x', [1, 2, 3])]) From 98373f0a1e38fee04e6d9737a1483b878f949f82 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Fri, 4 May 2018 22:50:59 +0100 Subject: [PATCH 093/282] xarray.dot is now based on da.einsum (#2089) * xarray.dot is now based on da.einsum * Merge what's new * Informative error if dask version too old * Tweak test when there's no dask or wrong dask version --- doc/whats-new.rst | 4 ++++ xarray/core/computation.py | 18 +++--------------- xarray/core/duck_array_ops.py | 23 +++++++++++++++-------- xarray/tests/test_computation.py | 31 ++++++++++++++++++------------- xarray/ufuncs.py | 2 +- 5 files changed, 41 insertions(+), 37 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 59aad702eea..8006c658e01 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,10 @@ Enhancements - Support writing lists of strings as netCDF attributes (:issue:`2044`). By `Dan Nowacki `_. +- :py:meth:`~xarray.dot` on dask-backed data will now call :func:`dask.array.einsum`. + This greatly boosts speed and allows chunking on the core dims. + The function now requires dask >= 0.17.3 to work on dask-backed data + (:issue:`2074`). By `Guido Imperiale `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 7e9e7273229..f06e90b583b 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -1014,15 +1014,6 @@ def dot(*arrays, **kwargs): output_core_dims = [tuple(d for d in all_dims if d not in dims + broadcast_dims)] - # we use tensordot if possible, because it is more efficient for dask - if len(broadcast_dims) == 0 and len(arrays) == 2: - axes = [[arr.get_axis_num(d) for d in arr.dims if d in dims] - for arr in arrays] - return apply_ufunc(duck_array_ops.tensordot, *arrays, dask='allowed', - input_core_dims=input_core_dims, - output_core_dims=output_core_dims, - kwargs={'axes': axes}) - # construct einsum subscripts, such as '...abc,...ab->...c' # Note: input_core_dims are always moved to the last position subscripts_list = ['...' + ''.join([dim_map[d] for d in ds]) for ds @@ -1030,16 +1021,13 @@ def dot(*arrays, **kwargs): subscripts = ','.join(subscripts_list) subscripts += '->...' + ''.join([dim_map[d] for d in output_core_dims[0]]) - # dtype estimation is necessary for dask='parallelized' - out_dtype = dtypes.result_type(*arrays) - # subscripts should be passed to np.einsum as arg, not as kwargs. We need - # to construct a partial function for parallelized computation. - func = functools.partial(np.einsum, subscripts) + # to construct a partial function for apply_ufunc to work. + func = functools.partial(duck_array_ops.einsum, subscripts) result = apply_ufunc(func, *arrays, input_core_dims=input_core_dims, output_core_dims=output_core_dims, - dask='parallelized', output_dtypes=[out_dtype]) + dask='allowed') return result.transpose(*[d for d in all_dims if d in result.dims]) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index faea30cdd99..3a2c123f87e 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -34,19 +34,24 @@ def _dask_or_eager_func(name, eager_module=np, dask_module=dask_array, - list_of_args=False, n_array_args=1): + list_of_args=False, array_args=slice(1), + requires_dask=None): """Create a function that dispatches to dask for dask array inputs.""" if dask_module is not None: def f(*args, **kwargs): if list_of_args: dispatch_args = args[0] else: - dispatch_args = args[:n_array_args] + dispatch_args = args[array_args] if any(isinstance(a, dask_array.Array) for a in dispatch_args): - module = dask_module + try: + wrapped = getattr(dask_module, name) + except AttributeError as e: + raise AttributeError("%s: requires dask >=%s" % + (e, requires_dask)) else: - module = eager_module - return getattr(module, name)(*args, **kwargs) + wrapped = getattr(eager_module, name) + return wrapped(*args, ** kwargs) else: def f(data, *args, **kwargs): return getattr(eager_module, name)(data, *args, **kwargs) @@ -80,9 +85,9 @@ def isnull(data): transpose = _dask_or_eager_func('transpose') -_where = _dask_or_eager_func('where', n_array_args=3) +_where = _dask_or_eager_func('where', array_args=slice(3)) isin = _dask_or_eager_func('isin', eager_module=npcompat, - dask_module=dask_array_compat, n_array_args=2) + dask_module=dask_array_compat, array_args=slice(2)) take = _dask_or_eager_func('take') broadcast_to = _dask_or_eager_func('broadcast_to') @@ -92,7 +97,9 @@ def isnull(data): array_all = _dask_or_eager_func('all') array_any = _dask_or_eager_func('any') -tensordot = _dask_or_eager_func('tensordot', n_array_args=2) +tensordot = _dask_or_eager_func('tensordot', array_args=slice(2)) +einsum = _dask_or_eager_func('einsum', array_args=slice(1, None), + requires_dask='0.17.3') def asarray(data): diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index 88710e55091..db10ee3e820 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -744,10 +744,14 @@ def test_vectorize_dask(): assert_identical(expected, actual) -@pytest.mark.parametrize('dask', [True, False]) -def test_dot(dask): - if not has_dask: - pytest.skip('test for dask.') +@pytest.mark.parametrize('use_dask', [True, False]) +def test_dot(use_dask): + if use_dask: + if not has_dask: + pytest.skip('test for dask.') + import dask + if LooseVersion(dask.__version__) < LooseVersion('0.17.3'): + pytest.skip("needs dask.array.einsum") a = np.arange(30 * 4).reshape(30, 4) b = np.arange(30 * 4 * 5).reshape(30, 4, 5) @@ -757,7 +761,7 @@ def test_dot(dask): da_b = xr.DataArray(b, dims=['a', 'b', 'c'], coords={'a': np.linspace(0, 1, 30)}) da_c = xr.DataArray(c, dims=['c', 'e']) - if dask: + if use_dask: da_a = da_a.chunk({'a': 3}) da_b = da_b.chunk({'a': 3}) da_c = da_c.chunk({'c': 3}) @@ -783,7 +787,7 @@ def test_dot(dask): assert (actual.data == np.einsum('ij,ijk->k', a, b)).all() assert isinstance(actual.data, type(da_a.variable.data)) - if dask: + if use_dask: da_a = da_a.chunk({'a': 3}) da_b = da_b.chunk({'a': 3}) actual = xr.dot(da_a, da_b, dims=['b']) @@ -791,10 +795,6 @@ def test_dot(dask): assert (actual.data == np.einsum('ij,ijk->ik', a, b)).all() assert isinstance(actual.variable.data, type(da_a.variable.data)) - pytest.skip('dot for dask array requires rechunking for core ' - 'dimensions.') - - # following requires rechunking actual = xr.dot(da_a, da_b, dims=['b']) assert actual.dims == ('a', 'c') assert (actual.data == np.einsum('ij,ijk->ik', a, b)).all() @@ -830,12 +830,17 @@ def test_dot(dask): assert actual.dims == ('b', ) assert (actual.data == np.einsum('ij->j ', a)).all() + # empty dim + actual = xr.dot(da_a.sel(a=[]), da_a.sel(a=[]), dims='a') + assert actual.dims == ('b', ) + assert (actual.data == np.zeros(actual.shape)).all() + with pytest.raises(TypeError): - actual = xr.dot(da_a, dims='a', invalid=None) + xr.dot(da_a, dims='a', invalid=None) with pytest.raises(TypeError): - actual = xr.dot(da_a.to_dataset(name='da'), dims='a') + xr.dot(da_a.to_dataset(name='da'), dims='a') with pytest.raises(TypeError): - actual = xr.dot(dims='a') + xr.dot(dims='a') def test_where(): diff --git a/xarray/ufuncs.py b/xarray/ufuncs.py index d9fcc1eac5d..628f8568a6d 100644 --- a/xarray/ufuncs.py +++ b/xarray/ufuncs.py @@ -50,7 +50,7 @@ def __call__(self, *args, **kwargs): 'directly.', PendingDeprecationWarning, stacklevel=2) new_args = args - f = _dask_or_eager_func(self._name, n_array_args=len(args)) + f = _dask_or_eager_func(self._name, array_args=slice(len(args))) if len(args) > 2 or len(args) == 0: raise TypeError('cannot handle %s arguments for %r' % (len(args), self._name)) From c6977f18341e3872ba69360045ee3ef8212eb1bf Mon Sep 17 00:00:00 2001 From: crusaderky Date: Tue, 8 May 2018 03:25:39 +0100 Subject: [PATCH 094/282] h5netcdf new API support (#1915) * Ignore dask scratch area * Public API for HDF5 support * Remove save_mfdataset_hdf5 * Replace to_hdf5 with to_netcdf(engine='h5netcdf-ng') * h5netcdf-ng -> h5netcdf-new * Trivial fixes * Functional implementation * stickler fixes * Reimplement as extra params for h5netcdf * Cosmetic tweaks * Bugfixes * More robust mixed-style encoding handling * Crash on mismatched encoding if check_encoding=True * Test check_encoding * stickler fix * Use parentheses instead of explicit continuation with \ --- .gitignore | 1 + doc/whats-new.rst | 5 +++ xarray/backends/api.py | 1 + xarray/backends/h5netcdf_.py | 73 ++++++++++++++++++++++---------- xarray/backends/netCDF4_.py | 7 +++- xarray/core/dataarray.py | 3 +- xarray/core/dataset.py | 7 ++++ xarray/tests/test_backends.py | 78 +++++++++++++++++++++++++++++++++++ 8 files changed, 149 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 70458f00648..92e488ed616 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ nosetests.xml .tags* .testmon* .pytest_cache +dask-worker-space/ # asv environments .asv diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8006c658e01..d614a23d0fc 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,11 @@ Enhancements - Support writing lists of strings as netCDF attributes (:issue:`2044`). By `Dan Nowacki `_. +- :py:meth:`~xarray.Dataset.to_netcdf(engine='h5netcdf')` now accepts h5py + encoding settings ``compression`` and ``compression_opts``, along with the + NetCDF4-Python style settings ``gzip=True`` and ``complevel``. + This allows using any compression plugin installed in hdf5, e.g. LZF + (:issue:`1536`). By `Guido Imperiale `_. - :py:meth:`~xarray.dot` on dask-backed data will now call :func:`dask.array.einsum`. This greatly boosts speed and allows chunking on the core dims. The function now requires dask >= 0.17.3 to work on dask-backed data diff --git a/xarray/backends/api.py b/xarray/backends/api.py index da4ef537a3a..b8cfa3c926a 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -741,6 +741,7 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, Engine to use when writing netCDF files. If not provided, the default engine is chosen based on available dependencies, with a preference for 'netcdf4' if writing to a file on disk. + See `Dataset.to_netcdf` for additional information. Examples -------- diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 7beda03308e..d34fa2d9267 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -45,21 +45,21 @@ def _read_attributes(h5netcdf_var): # to ensure conventions decoding works properly on Python 3, decode all # bytes attributes to strings attrs = OrderedDict() - for k in h5netcdf_var.ncattrs(): - v = h5netcdf_var.getncattr(k) + for k, v in h5netcdf_var.attrs.items(): if k not in ['_FillValue', 'missing_value']: v = maybe_decode_bytes(v) attrs[k] = v return attrs -_extract_h5nc_encoding = functools.partial(_extract_nc4_variable_encoding, - lsd_okay=False, backend='h5netcdf') +_extract_h5nc_encoding = functools.partial( + _extract_nc4_variable_encoding, + lsd_okay=False, h5py_okay=True, backend='h5netcdf') def _open_h5netcdf_group(filename, mode, group): - import h5netcdf.legacyapi - ds = h5netcdf.legacyapi.Dataset(filename, mode=mode) + import h5netcdf + ds = h5netcdf.File(filename, mode=mode) with close_on_error(ds): return _nc4_group(ds, group, mode) @@ -96,10 +96,19 @@ def open_store_variable(self, name, var): attrs = _read_attributes(var) # netCDF4 specific encoding - encoding = dict(var.filters()) - chunking = var.chunking() - encoding['chunksizes'] = chunking \ - if chunking != 'contiguous' else None + encoding = { + 'chunksizes': var.chunks, + 'fletcher32': var.fletcher32, + 'shuffle': var.shuffle, + } + # Convert h5py-style compression options to NetCDF4-Python + # style, if possible + if var.compression == 'gzip': + encoding['zlib'] = True + encoding['complevel'] = var.compression_opts + elif var.compression is not None: + encoding['compression'] = var.compression + encoding['compression_opts'] = var.compression_opts # save source so __repr__ can detect if it's local or not encoding['source'] = self._filename @@ -130,14 +139,14 @@ def get_encoding(self): def set_dimension(self, name, length, is_unlimited=False): with self.ensure_open(autoclose=False): if is_unlimited: - self.ds.createDimension(name, size=None) + self.ds.dimensions[name] = None self.ds.resize_dimension(name, length) else: - self.ds.createDimension(name, size=length) + self.ds.dimensions[name] = length def set_attribute(self, key, value): with self.ensure_open(autoclose=False): - self.ds.setncattr(key, value) + self.ds.attrs[key] = value def encode_variable(self, variable): return _encode_nc4_variable(variable) @@ -149,8 +158,8 @@ def prepare_variable(self, name, variable, check_encoding=False, attrs = variable.attrs.copy() dtype = _get_datatype(variable) - fill_value = attrs.pop('_FillValue', None) - if dtype is str and fill_value is not None: + fillvalue = attrs.pop('_FillValue', None) + if dtype is str and fillvalue is not None: raise NotImplementedError( 'h5netcdf does not yet support setting a fill value for ' 'variable-length strings ' @@ -166,18 +175,38 @@ def prepare_variable(self, name, variable, check_encoding=False, raise_on_invalid=check_encoding) kwargs = {} - for key in ['zlib', 'complevel', 'shuffle', - 'chunksizes', 'fletcher32']: + # Convert from NetCDF4-Python style compression settings to h5py style + # If both styles are used together, h5py takes precedence + # If set_encoding=True, raise ValueError in case of mismatch + if encoding.pop('zlib', False): + if (check_encoding and encoding.get('compression') + not in (None, 'gzip')): + raise ValueError("'zlib' and 'compression' encodings mismatch") + encoding.setdefault('compression', 'gzip') + + if (check_encoding and encoding.get('complevel') not in + (None, encoding.get('compression_opts'))): + raise ValueError("'complevel' and 'compression_opts' encodings " + "mismatch") + complevel = encoding.pop('complevel', 0) + if complevel != 0: + encoding.setdefault('compression_opts', complevel) + + encoding['chunks'] = encoding.pop('chunksizes', None) + + for key in ['compression', 'compression_opts', 'shuffle', + 'chunks', 'fletcher32']: if key in encoding: kwargs[key] = encoding[key] - if name not in self.ds.variables: - nc4_var = self.ds.createVariable(name, dtype, variable.dims, - fill_value=fill_value, **kwargs) + if name not in self.ds: + nc4_var = self.ds.create_variable( + name, dtype=dtype, dimensions=variable.dims, + fillvalue=fillvalue, **kwargs) else: - nc4_var = self.ds.variables[name] + nc4_var = self.ds[name] for k, v in iteritems(attrs): - nc4_var.setncattr(k, v) + nc4_var.attrs[k] = v target = H5NetCDFArrayWrapper(name, self) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 1195301825b..a0f6cbcdd33 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -159,8 +159,8 @@ def _force_native_endianness(var): def _extract_nc4_variable_encoding(variable, raise_on_invalid=False, - lsd_okay=True, backend='netCDF4', - unlimited_dims=None): + lsd_okay=True, h5py_okay=False, + backend='netCDF4', unlimited_dims=None): if unlimited_dims is None: unlimited_dims = () @@ -171,6 +171,9 @@ def _extract_nc4_variable_encoding(variable, raise_on_invalid=False, 'chunksizes', 'shuffle', '_FillValue']) if lsd_okay: valid_encodings.add('least_significant_digit') + if h5py_okay: + valid_encodings.add('compression') + valid_encodings.add('compression_opts') if not raise_on_invalid and encoding.get('chunksizes') is not None: # It's possible to get encoded chunksizes larger than a dimension size diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 9ff631e7cfc..1ceaced5961 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1443,8 +1443,7 @@ def to_masked_array(self, copy=True): return np.ma.MaskedArray(data=self.values, mask=isnull, copy=copy) def to_netcdf(self, *args, **kwargs): - """ - Write DataArray contents to a netCDF file. + """Write DataArray contents to a netCDF file. Parameters ---------- diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index f28e7980b34..32913127636 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1123,6 +1123,13 @@ def to_netcdf(self, path=None, mode='w', format=None, group=None, variable specific encodings as values, e.g., ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1, 'zlib': True}, ...}`` + + The `h5netcdf` engine supports both the NetCDF4-style compression + encoding parameters ``{'zlib': True, 'complevel': 9}`` and the h5py + ones ``{'compression': 'gzip', 'compression_opts': 9}``. + This allows using any compression plugin installed in the HDF5 + library, e.g. LZF. + unlimited_dims : sequence of str, optional Dimension(s) that should be serialized as unlimited dimensions. By default, no dimensions are treated as unlimited dimensions. diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 7f8a440ba5d..632145007e2 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1668,6 +1668,84 @@ def test_encoding_unlimited_dims(self): self.assertEqual(actual.encoding['unlimited_dims'], set('y')) assert_equal(ds, actual) + def test_compression_encoding_h5py(self): + ENCODINGS = ( + # h5py style compression with gzip codec will be converted to + # NetCDF4-Python style on round-trip + ({'compression': 'gzip', 'compression_opts': 9}, + {'zlib': True, 'complevel': 9}), + # What can't be expressed in NetCDF4-Python style is + # round-tripped unaltered + ({'compression': 'lzf', 'compression_opts': None}, + {'compression': 'lzf', 'compression_opts': None}), + # If both styles are used together, h5py format takes precedence + ({'compression': 'lzf', 'compression_opts': None, + 'zlib': True, 'complevel': 9}, + {'compression': 'lzf', 'compression_opts': None})) + + for compr_in, compr_out in ENCODINGS: + data = create_test_data() + compr_common = { + 'chunksizes': (5, 5), + 'fletcher32': True, + 'shuffle': True, + 'original_shape': data.var2.shape + } + data['var2'].encoding.update(compr_in) + data['var2'].encoding.update(compr_common) + compr_out.update(compr_common) + with self.roundtrip(data) as actual: + for k, v in compr_out.items(): + self.assertEqual(v, actual['var2'].encoding[k]) + + def test_compression_check_encoding_h5py(self): + """When mismatched h5py and NetCDF4-Python encodings are expressed + in to_netcdf(encoding=...), must raise ValueError + """ + data = Dataset({'x': ('y', np.arange(10.0))}) + # Compatible encodings are graciously supported + with create_tmp_file() as tmp_file: + data.to_netcdf( + tmp_file, engine='h5netcdf', + encoding={'x': {'compression': 'gzip', 'zlib': True, + 'compression_opts': 6, 'complevel': 6}}) + with open_dataset(tmp_file, engine='h5netcdf') as actual: + assert actual.x.encoding['zlib'] is True + assert actual.x.encoding['complevel'] == 6 + + # Incompatible encodings cause a crash + with create_tmp_file() as tmp_file: + with raises_regex(ValueError, + "'zlib' and 'compression' encodings mismatch"): + data.to_netcdf( + tmp_file, engine='h5netcdf', + encoding={'x': {'compression': 'lzf', 'zlib': True}}) + + with create_tmp_file() as tmp_file: + with raises_regex( + ValueError, + "'complevel' and 'compression_opts' encodings mismatch"): + data.to_netcdf( + tmp_file, engine='h5netcdf', + encoding={'x': {'compression': 'gzip', + 'compression_opts': 5, 'complevel': 6}}) + + def test_dump_encodings_h5py(self): + # regression test for #709 + ds = Dataset({'x': ('y', np.arange(10.0))}) + + kwargs = {'encoding': {'x': { + 'compression': 'gzip', 'compression_opts': 9}}} + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + self.assertEqual(actual.x.encoding['zlib'], True) + self.assertEqual(actual.x.encoding['complevel'], 9) + + kwargs = {'encoding': {'x': { + 'compression': 'lzf', 'compression_opts': None}}} + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + self.assertEqual(actual.x.encoding['compression'], 'lzf') + self.assertEqual(actual.x.encoding['compression_opts'], None) + # tests pending h5netcdf fix @unittest.skip From aeae80b21ed659e14d4378a513f2351452eed460 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Tue, 8 May 2018 00:23:02 -0400 Subject: [PATCH 095/282] DOC: Add resample e.g. Edit rolling e.g. Add groupby e.g. (#2101) * DOC: Add resample e.g. Edit rolling e.g. Add groupby e.g. * DOC: Add 2d resample example * DOC: Add upsample example in resample * DOC: drop sentence is resample docstring * extend resample DeprecationWarning. Drop n-d resample example. * change resample DeprecationWarning * don't display how twice in the warning * DOC: add assign_coords example * DOC: remove parameters to resample --- xarray/core/common.py | 95 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 15 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index 5beb5234d4c..0c6e0fccd8e 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -308,6 +308,25 @@ def assign_coords(self, **kwargs): assigned : same type as caller A new object with the new coordinates in addition to the existing data. + + Examples + -------- + + Convert longitude coordinates from 0-359 to -180-179: + + >>> da = xr.DataArray(np.random.rand(4), + ... coords=[np.array([358, 359, 0, 1])], + ... dims='lon') + >>> da + + array([0.28298 , 0.667347, 0.657938, 0.177683]) + Coordinates: + * lon (lon) int64 358 359 0 1 + >>> da.assign_coords(lon=(((da.lon + 180) % 360) - 180)) + + array([0.28298 , 0.667347, 0.657938, 0.177683]) + Coordinates: + * lon (lon) int64 -2 -1 0 1 Notes ----- @@ -426,7 +445,27 @@ def groupby(self, group, squeeze=True): grouped : GroupBy A `GroupBy` object patterned after `pandas.GroupBy` that can be iterated over in the form of `(unique_value, grouped_array)` pairs. - + + Examples + -------- + Calculate daily anomalies for daily data: + + >>> da = xr.DataArray(np.linspace(0, 1826, num=1827), + ... coords=[pd.date_range('1/1/2000', '31/12/2004', + ... freq='D')], + ... dims='time') + >>> da + + array([0.000e+00, 1.000e+00, 2.000e+00, ..., 1.824e+03, 1.825e+03, 1.826e+03]) + Coordinates: + * time (time) datetime64[ns] 2000-01-01 2000-01-02 2000-01-03 ... + >>> da.groupby('time.dayofyear') - da.groupby('time.dayofyear').mean('time') + + array([-730.8, -730.8, -730.8, ..., 730.2, 730.2, 730.5]) + Coordinates: + * time (time) datetime64[ns] 2000-01-01 2000-01-02 2000-01-03 ... + dayofyear (time) int64 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 ... + See Also -------- core.groupby.DataArrayGroupBy @@ -514,7 +553,7 @@ def rolling(self, min_periods=None, center=False, **windows): -------- Create rolling seasonal average of monthly data e.g. DJF, JFM, ..., SON: - >>> da = xr.DataArray(np.linspace(0,11,num=12), + >>> da = xr.DataArray(np.linspace(0, 11, num=12), ... coords=[pd.date_range('15/12/1999', ... periods=12, freq=pd.DateOffset(months=1))], ... dims='time') @@ -523,19 +562,19 @@ def rolling(self, min_periods=None, center=False, **windows): array([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.]) Coordinates: * time (time) datetime64[ns] 1999-12-15 2000-01-15 2000-02-15 ... - >>> da.rolling(time=3).mean() + >>> da.rolling(time=3, center=True).mean() - array([ nan, nan, 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.]) + array([nan, 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., nan]) Coordinates: * time (time) datetime64[ns] 1999-12-15 2000-01-15 2000-02-15 ... Remove the NaNs using ``dropna()``: - >>> da.rolling(time=3).mean().dropna('time') + >>> da.rolling(time=3, center=True).mean().dropna('time') - array([ 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.]) + array([ 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.]) Coordinates: - * time (time) datetime64[ns] 2000-02-15 2000-03-15 2000-04-15 ... + * time (time) datetime64[ns] 2000-01-15 2000-02-15 2000-03-15 ... See Also -------- @@ -550,9 +589,8 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, closed=None, label=None, base=0, keep_attrs=False, **indexer): """Returns a Resample object for performing resampling operations. - Handles both downsampling and upsampling. Upsampling with filling is - not supported; if any intervals contain no values from the original - object, they will be given the value ``NaN``. + Handles both downsampling and upsampling. If any intervals contain no + values from the original object, they will be given the value ``NaN``. Parameters ---------- @@ -578,7 +616,34 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, ------- resampled : same type as caller This object resampled. - + + Examples + -------- + Downsample monthly time-series data to seasonal data: + + >>> da = xr.DataArray(np.linspace(0, 11, num=12), + ... coords=[pd.date_range('15/12/1999', + ... periods=12, freq=pd.DateOffset(months=1))], + ... dims='time') + >>> da + + array([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.]) + Coordinates: + * time (time) datetime64[ns] 1999-12-15 2000-01-15 2000-02-15 ... + >>> da.resample(time="Q-DEC").mean() + + array([ 1., 4., 7., 10.]) + Coordinates: + * time (time) datetime64[ns] 2000-02-29 2000-05-31 2000-08-31 2000-11-30 + + Upsample monthly time-series data to daily data: + + >>> da.resample(time='1D').interpolate('linear') + + array([ 0. , 0.032258, 0.064516, ..., 10.935484, 10.967742, 11. ]) + Coordinates: + * time (time) datetime64[ns] 1999-12-15 1999-12-16 1999-12-17 ... + References ---------- @@ -628,10 +693,10 @@ def _resample_immediately(self, freq, dim, how, skipna, warnings.warn("\n.resample() has been modified to defer " "calculations. Instead of passing 'dim' and " - "'how=\"{how}\", instead consider using " - ".resample({dim}=\"{freq}\").{how}() ".format( - dim=dim, freq=freq, how=how - ), DeprecationWarning, stacklevel=3) + "how=\"{how}\", instead consider using " + ".resample({dim}=\"{freq}\").{how}('{dim}') ".format( + dim=dim, freq=freq, how=how), + DeprecationWarning, stacklevel=3) if isinstance(dim, basestring): dim = self[dim] From c046528522a6d4cf18c81a19aeae82f5f7d63d34 Mon Sep 17 00:00:00 2001 From: Henk Griffioen Date: Wed, 9 May 2018 17:28:32 +0200 Subject: [PATCH 096/282] DOC: Update link to documentation of Rasterio (#2110) --- doc/io.rst | 2 +- doc/whats-new.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/io.rst b/doc/io.rst index c14e1516b38..668416e714d 100644 --- a/doc/io.rst +++ b/doc/io.rst @@ -534,7 +534,7 @@ longitudes and latitudes. considered as being experimental. Please report any bug you may find on xarray's github repository. -.. _rasterio: https://mapbox.github.io/rasterio/ +.. _rasterio: https://rasterio.readthedocs.io/en/latest/ .. _test files: https://github.com/mapbox/rasterio/blob/master/tests/data/RGB.byte.tif .. _pyproj: https://github.com/jswhit/pyproj diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d614a23d0fc..fdbe6831a24 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -785,7 +785,7 @@ Enhancements By `Stephan Hoyer `_. - New function :py:func:`~xarray.open_rasterio` for opening raster files with - the `rasterio `_ library. + the `rasterio `_ library. See :ref:`the docs ` for details. By `Joe Hamman `_, `Nic Wayand `_ and From 70e2eb539d2fe33ee1b5efbd5d2476649dea898b Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 9 May 2018 17:45:40 -0700 Subject: [PATCH 097/282] Plotting upgrades (#2092) * Support xincrease, yincrease * Better tick label rotation in case of dateticks. Avoid autofmt_xdate because it deletes all x-axis ticklabels except for the last subplot. * Tests * docs. * review. * Prevent unclosed file ResourceWarning. --- doc/plotting.rst | 10 ++++++++++ doc/whats-new.rst | 4 ++++ xarray/plot/plot.py | 21 ++++++++++++++++++--- xarray/plot/utils.py | 1 + xarray/tests/test_plot.py | 7 +++++++ 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index c85a54d783b..28fbe7062a6 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -207,6 +207,16 @@ It is also possible to make line plots such that the data are on the x-axis and @savefig plotting_example_xy_kwarg.png air.isel(time=10, lon=[10, 11]).plot.line(y='lat', hue='lon') +Changing Axes Direction +----------------------- + +The keyword arguments ``xincrease`` and ``yincrease`` let you control the axes direction. + +.. ipython:: python + + @savefig plotting_example_xincrease_yincrease_kwarg.png + air.isel(time=10, lon=[10, 11]).plot.line(y='lat', hue='lon', xincrease=False, yincrease=False) + Two Dimensions -------------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fdbe6831a24..3c2e143bec3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -55,6 +55,10 @@ Bug fixes By `Keisuke Fujii `_. - Better error handling in ``open_mfdataset`` (:issue:`2077`). By `Stephan Hoyer `_. +- ``plot.line()`` does not call ``autofmt_xdate()`` anymore. Instead it changes the rotation and horizontal alignment of labels without removing the x-axes of any other subplots in the figure (if any). + By `Deepak Cherian `_. +- ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change the direction of the respective axes. + By `Deepak Cherian `_. .. _whats-new.0.10.3: diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 94ddc8c0535..6a3bed08f72 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -182,6 +182,12 @@ def line(darray, *args, **kwargs): Coordinates for x, y axis. Only one of these may be specified. The other coordinate plots values from the DataArray on which this plot method is called. + xincrease : None, True, or False, optional + Should the values on the x axes be increasing from left to right? + if None, use the default for the matplotlib function. + yincrease : None, True, or False, optional + Should the values on the y axes be increasing from top to bottom? + if None, use the default for the matplotlib function. add_legend : boolean, optional Add legend with y axis coordinates (2D inputs only). *args, **kwargs : optional @@ -203,6 +209,8 @@ def line(darray, *args, **kwargs): hue = kwargs.pop('hue', None) x = kwargs.pop('x', None) y = kwargs.pop('y', None) + xincrease = kwargs.pop('xincrease', True) + yincrease = kwargs.pop('yincrease', True) add_legend = kwargs.pop('add_legend', True) ax = get_axis(figsize, size, aspect, ax) @@ -269,8 +277,15 @@ def line(darray, *args, **kwargs): title=huelabel) # Rotate dates on xlabels + # Do this without calling autofmt_xdate so that x-axes ticks + # on other subplots (if any) are not deleted. + # https://stackoverflow.com/questions/17430105/autofmt-xdate-deletes-x-axis-labels-of-all-subplots if np.issubdtype(xplt.dtype, np.datetime64): - ax.get_figure().autofmt_xdate() + for xlabels in ax.get_xticklabels(): + xlabels.set_rotation(30) + xlabels.set_ha('right') + + _update_axes_limits(ax, xincrease, yincrease) return primitive @@ -429,10 +444,10 @@ def _plot2d(plotfunc): Use together with ``col`` to wrap faceted plots xincrease : None, True, or False, optional Should the values on the x axes be increasing from left to right? - if None, use the default for the matplotlib function + if None, use the default for the matplotlib function. yincrease : None, True, or False, optional Should the values on the y axes be increasing from top to bottom? - if None, use the default for the matplotlib function + if None, use the default for the matplotlib function. add_colorbar : Boolean, optional Adds colorbar to axis add_labels : Boolean, optional diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 59d67ed79f1..3db8bcab3a7 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -21,6 +21,7 @@ def _load_default_cmap(fname='default_colormap.csv'): # Not sure what the first arg here should be f = pkg_resources.resource_stream(__name__, fname) cm_data = pd.read_csv(f, header=None).values + f.close() return LinearSegmentedColormap.from_list('viridis', cm_data) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 2a5eeb86bdd..a3446fe240b 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -352,6 +352,13 @@ def test_x_ticks_are_rotated_for_time(self): rotation = plt.gca().get_xticklabels()[0].get_rotation() assert rotation != 0 + def test_xyincrease_false_changes_axes(self): + self.darray.plot.line(xincrease=False, yincrease=False) + xlim = plt.gca().get_xlim() + ylim = plt.gca().get_ylim() + diffs = xlim[1] - xlim[0], ylim[1] - ylim[0] + assert all(x < 0 for x in diffs) + def test_slice_in_title(self): self.darray.coords['d'] = 10 self.darray.plot.line() From 6d8ac11ca0a785a6fe176eeca9b735c321a35527 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Thu, 10 May 2018 11:49:59 -0600 Subject: [PATCH 098/282] Fix docstring formatting for load(). (#2115) Need '::' to introduce a code literal block. This was causing MetPy's doc build to warn (since we inherit AbstractDataStore). --- xarray/backends/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/common.py b/xarray/backends/common.py index c46f9d5b552..7d8aa8446a2 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -198,7 +198,7 @@ def load(self): A centralized loading function makes it easier to create data stores that do automatic encoding/decoding. - For example: + For example:: class SuffixAppendingDataStore(AbstractDataStore): From d63001cdbc3bd84f4d6d90bd570a2215ea9e5c2e Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sat, 12 May 2018 07:54:43 +0900 Subject: [PATCH 099/282] Support keep_attrs for apply_ufunc for xr.Variable (#2119) * Support keep_attrs for apply_ufunc for xr.Dataset, xr.Variable * whats new * whats new again * improve doc --- doc/whats-new.rst | 3 +++ xarray/core/computation.py | 21 +++++++++++---------- xarray/tests/test_computation.py | 7 +++++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3c2e143bec3..b177fc702c0 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -49,6 +49,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed a bug where `keep_attrs=True` flag was neglected if + :py:func:`apply_func` was used with :py:class:`Variable`. (:issue:`2114`) + By `Keisuke Fujii `_. - When assigning a :py:class:`DataArray` to :py:class:`Dataset`, any conflicted non-dimensional coordinates of the DataArray are now dropped. (:issue:`2068`) diff --git a/xarray/core/computation.py b/xarray/core/computation.py index f06e90b583b..77a52ac055d 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -195,7 +195,6 @@ def apply_dataarray_ufunc(func, *args, **kwargs): signature = kwargs.pop('signature') join = kwargs.pop('join', 'inner') exclude_dims = kwargs.pop('exclude_dims', _DEFAULT_FROZEN_SET) - keep_attrs = kwargs.pop('keep_attrs', False) if kwargs: raise TypeError('apply_dataarray_ufunc() got unexpected keyword ' 'arguments: %s' % list(kwargs)) @@ -217,11 +216,6 @@ def apply_dataarray_ufunc(func, *args, **kwargs): coords, = result_coords out = DataArray(result_var, coords, name=name, fastpath=True) - if keep_attrs and isinstance(args[0], DataArray): - if isinstance(out, tuple): - out = tuple(ds._copy_attrs_from(args[0]) for ds in out) - else: - out._copy_attrs_from(args[0]) return out @@ -526,6 +520,7 @@ def apply_variable_ufunc(func, *args, **kwargs): dask = kwargs.pop('dask', 'forbidden') output_dtypes = kwargs.pop('output_dtypes', None) output_sizes = kwargs.pop('output_sizes', None) + keep_attrs = kwargs.pop('keep_attrs', False) if kwargs: raise TypeError('apply_variable_ufunc() got unexpected keyword ' 'arguments: %s' % list(kwargs)) @@ -567,11 +562,17 @@ def func(*arrays): if signature.num_outputs > 1: output = [] for dims, data in zip(output_dims, result_data): - output.append(Variable(dims, data)) + var = Variable(dims, data) + if keep_attrs and isinstance(args[0], Variable): + var.attrs.update(args[0].attrs) + output.append(var) return tuple(output) else: dims, = output_dims - return Variable(dims, result_data) + var = Variable(dims, result_data) + if keep_attrs and isinstance(args[0], Variable): + var.attrs.update(args[0].attrs) + return var def _apply_with_dask_atop(func, args, input_dims, output_dims, signature, @@ -902,6 +903,7 @@ def earth_mover_distance(first_samples, variables_ufunc = functools.partial(apply_variable_ufunc, func, signature=signature, exclude_dims=exclude_dims, + keep_attrs=keep_attrs, dask=dask, output_dtypes=output_dtypes, output_sizes=output_sizes) @@ -930,8 +932,7 @@ def earth_mover_distance(first_samples, return apply_dataarray_ufunc(variables_ufunc, *args, signature=signature, join=join, - exclude_dims=exclude_dims, - keep_attrs=keep_attrs) + exclude_dims=exclude_dims) elif any(isinstance(a, Variable) for a in args): return variables_ufunc(*args) else: diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index db10ee3e820..c84ed17bfd3 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -480,12 +480,19 @@ def add(a, b, keep_attrs): a = xr.DataArray([0, 1], [('x', [0, 1])]) a.attrs['attr'] = 'da' + a['x'].attrs['attr'] = 'da_coord' b = xr.DataArray([1, 2], [('x', [0, 1])]) actual = add(a, b, keep_attrs=False) assert not actual.attrs actual = add(a, b, keep_attrs=True) assert_identical(actual.attrs, a.attrs) + assert_identical(actual['x'].attrs, a['x'].attrs) + + actual = add(a.variable, b.variable, keep_attrs=False) + assert not actual.attrs + actual = add(a.variable, b.variable, keep_attrs=True) + assert_identical(actual.attrs, a.attrs) a = xr.Dataset({'x': [0, 1]}) a.attrs['attr'] = 'ds' From a52540505f606bd7619536d82d43f19f2cbe58b5 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Sat, 12 May 2018 15:15:54 +0900 Subject: [PATCH 100/282] Fixes centerized rolling with bottleneck. Also, fixed rolling with an integer dask array. (#2122) --- doc/whats-new.rst | 3 +++ xarray/core/dask_array_ops.py | 5 ++++- xarray/core/rolling.py | 16 ++++++++++++---- xarray/tests/test_dataarray.py | 30 +++++++++++++++++++++++++----- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b177fc702c0..cc16991ccf1 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -49,6 +49,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed a bug in `rolling` with bottleneck. Also, fixed a bug in rolling an + integer dask array. (:issue:`21133`) + By `Keisuke Fujii `_. - Fixed a bug where `keep_attrs=True` flag was neglected if :py:func:`apply_func` was used with :py:class:`Variable`. (:issue:`2114`) By `Keisuke Fujii `_. diff --git a/xarray/core/dask_array_ops.py b/xarray/core/dask_array_ops.py index 4bd3766ced9..ee87c3564cc 100644 --- a/xarray/core/dask_array_ops.py +++ b/xarray/core/dask_array_ops.py @@ -3,6 +3,7 @@ import numpy as np from . import nputils +from . import dtypes try: import dask.array as da @@ -12,12 +13,14 @@ def dask_rolling_wrapper(moving_func, a, window, min_count=None, axis=-1): '''wrapper to apply bottleneck moving window funcs on dask arrays''' + dtype, fill_value = dtypes.maybe_promote(a.dtype) + a = a.astype(dtype) # inputs for ghost if axis < 0: axis = a.ndim + axis depth = {d: 0 for d in range(a.ndim)} depth[axis] = window - 1 - boundary = {d: np.nan for d in range(a.ndim)} + boundary = {d: fill_value for d in range(a.ndim)} # create ghosted arrays ag = da.ghost.ghost(a, depth=depth, boundary=boundary) # apply rolling func diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 079c60f35a7..f54a4c36631 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -285,18 +285,26 @@ def wrapped_func(self, **kwargs): padded = self.obj.variable if self.center: - shift = (-self.window // 2) + 1 - if (LooseVersion(np.__version__) < LooseVersion('1.13') and self.obj.dtype.kind == 'b'): # with numpy < 1.13 bottleneck cannot handle np.nan-Boolean # mixed array correctly. We cast boolean array to float. padded = padded.astype(float) + + if isinstance(padded.data, dask_array_type): + # Workaround to make the padded chunk size is larger than + # self.window-1 + shift = - (self.window - 1) + offset = -shift - self.window // 2 + valid = (slice(None), ) * axis + ( + slice(offset, offset + self.obj.shape[axis]), ) + else: + shift = (-self.window // 2) + 1 + valid = (slice(None), ) * axis + (slice(-shift, None), ) padded = padded.pad_with_fill_value(**{self.dim: (0, -shift)}) - valid = (slice(None), ) * axis + (slice(-shift, None), ) if isinstance(padded.data, dask_array_type): - values = dask_rolling_wrapper(func, self.obj.data, + values = dask_rolling_wrapper(func, padded, window=self.window, min_count=min_count, axis=axis) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 32ab3a634cb..e9a2babfa2e 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3439,23 +3439,43 @@ def test_rolling_wrapped_bottleneck(da, name, center, min_periods): assert_equal(actual, da['time']) -@pytest.mark.parametrize('name', ('sum', 'mean', 'std', 'min', 'max', - 'median')) +@pytest.mark.parametrize('name', ('mean', 'count')) @pytest.mark.parametrize('center', (True, False, None)) @pytest.mark.parametrize('min_periods', (1, None)) -def test_rolling_wrapped_bottleneck_dask(da_dask, name, center, min_periods): +@pytest.mark.parametrize('window', (7, 8)) +def test_rolling_wrapped_dask(da_dask, name, center, min_periods, window): pytest.importorskip('dask.array') # dask version - rolling_obj = da_dask.rolling(time=7, min_periods=min_periods) + rolling_obj = da_dask.rolling(time=window, min_periods=min_periods, + center=center) actual = getattr(rolling_obj, name)().load() # numpy version - rolling_obj = da_dask.load().rolling(time=7, min_periods=min_periods) + rolling_obj = da_dask.load().rolling(time=window, min_periods=min_periods, + center=center) expected = getattr(rolling_obj, name)() # using all-close because rolling over ghost cells introduces some # precision errors assert_allclose(actual, expected) + # with zero chunked array GH:2113 + rolling_obj = da_dask.chunk().rolling(time=window, min_periods=min_periods, + center=center) + actual = getattr(rolling_obj, name)().load() + assert_allclose(actual, expected) + + +@pytest.mark.parametrize('center', (True, None)) +def test_rolling_wrapped_dask_nochunk(center): + # GH:2113 + pytest.importorskip('dask.array') + + da_day_clim = xr.DataArray(np.arange(1, 367), + coords=[np.arange(1, 367)], dims='dayofyear') + expected = da_day_clim.rolling(dayofyear=31, center=center).mean() + actual = da_day_clim.chunk().rolling(dayofyear=31, center=center).mean() + assert_allclose(actual, expected) + @pytest.mark.parametrize('center', (True, False)) @pytest.mark.parametrize('min_periods', (None, 1, 2, 3)) From 2c6bd2d1b09a84488ab1f1ebffa9cd359d0437ce Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 11 May 2018 23:36:36 -0700 Subject: [PATCH 101/282] Prevent Inf from screwing colorbar scale. (#2120) pd.isnull([np.inf]) is True while np.isfinite([np.inf]) is False. Let's use the latter. --- doc/whats-new.rst | 2 ++ xarray/plot/utils.py | 2 +- xarray/tests/test_plot.py | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index cc16991ccf1..cc5506b553c 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -65,6 +65,8 @@ Bug fixes By `Deepak Cherian `_. - ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change the direction of the respective axes. By `Deepak Cherian `_. +- Colorbar limits are now determined by excluding ±Infs too. + By `Deepak Cherian `_. .. _whats-new.0.10.3: diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 3db8bcab3a7..7ba48819518 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -160,7 +160,7 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, """ import matplotlib as mpl - calc_data = np.ravel(plot_data[~pd.isnull(plot_data)]) + calc_data = np.ravel(plot_data[np.isfinite(plot_data)]) # Handle all-NaN input data gracefully if calc_data.size == 0: diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index a3446fe240b..aadc452b8a7 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -426,6 +426,15 @@ def test_center(self): assert cmap_params['levels'] is None assert cmap_params['norm'] is None + def test_nan_inf_are_ignored(self): + cmap_params1 = _determine_cmap_params(self.data) + data = self.data + data[50:55] = np.nan + data[56:60] = np.inf + cmap_params2 = _determine_cmap_params(data) + assert cmap_params1['vmin'] == cmap_params2['vmin'] + assert cmap_params1['vmax'] == cmap_params2['vmax'] + @pytest.mark.slow def test_integer_levels(self): data = self.data + 1 From ebe0dd03187a5c3138ea12ca4beb13643679fe21 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 13 May 2018 01:19:09 -0400 Subject: [PATCH 102/282] CFTimeIndex (#1252) * Start on implementing and testing NetCDFTimeIndex * TST Move to using pytest fixtures to structure tests * Address initial review comments * Address second round of review comments * Fix failing python3 tests * Match test method name to method name * First attempts at integrating NetCDFTimeIndex into xarray This is a first pass at the following: - Resetting the logic for decoding datetimes such that `np.datetime64` objects are never used for non-standard calendars - Adding logic to use a `NetCDFTimeIndex` whenever `netcdftime.datetime` objects are used in an array being cast as an index (so if one reads in a Dataset from a netCDF file or creates one in Python, which is indexed by a time coordinate that uses `netcdftime.datetime` objects a NetCDFTimeIndex will be used rather than a generic object-based index) - Adding logic to encode `netcdftime.datetime` objects when saving out to netCDF files * Cleanup * Fix DataFrame and Series test failures for NetCDFTimeIndex These were related to a recent minor upstream change in pandas: https://github.com/pandas-dev/pandas/blame/master/pandas/core/indexing.py#L1433 * First pass at making NetCDFTimeIndex compatible with #1356 * Address initial review comments * Restore test_conventions.py * Fix failing test in test_utils.py * flake8 * Update for standalone netcdftime * Address stickler-ci comments * Skip test_format_netcdftime_datetime if netcdftime not installed * A start on documentation * Fix failing zarr tests related to netcdftime encoding * Simplify test_decode_standard_calendar_single_element_non_ns_range * Address a couple review comments * Use else clause in _maybe_cast_to_netcdftimeindex * Start on adding enable_netcdftimeindex option * Continue parametrizing tests in test_coding_times.py * Update time-series.rst for enable_netcdftimeindex option * Use :py:func: in rst for xarray.set_options * Add a what's new entry and test that resample raises a TypeError * Move what's new entry to the version 0.10.3 section * Add version-dependent pathway for importing netcdftime.datetime * Make NetCDFTimeIndex and date decoding/encoding compatible with datetime.datetime * Remove logic to make NetCDFTimeIndex compatible with datetime.datetime * Documentation edits * Ensure proper enable_netcdftimeindex option is used under lazy decoding Prior to this, opening a dataset with enable_netcdftimeindex set to True and then accessing one of its variables outside the context manager would lead to it being decoded with the default enable_netcdftimeindex (which is False). This makes sure that lazy decoding takes into account the context under which it was called. * Add fix and test for concatenating variables with a NetCDFTimeIndex Previously when concatenating variables indexed by a NetCDFTimeIndex the index would be wrongly converted to a generic pd.Index * Further namespace changes due to netcdftime/cftime renaming * NetCDFTimeIndex -> CFTimeIndex * Documentation updates * Only allow use of CFTimeIndex when using the standalone cftime Also only allow for serialization of cftime.datetime objects when using the standalone cftime package. * Fix errant what's new changes * flake8 * Fix skip logic in test_cftimeindex.py * Use only_use_cftime_datetimes option in num2date * Require standalone cftime library for all new functionality Add tests/fixes for dt accessor with cftime datetimes * Improve skipping logic in test_cftimeindex.py * Fix skipping logic in test_cftimeindex.py for when cftime or netcdftime are not available. Use existing requires_cftime decorator where possible (i.e. only on tests that are not parametrized via pytest.mark.parametrize) * Fix skip logic in Python 3.4 build for test_cftimeindex.py * Improve error messages when for when the standalone cftime is not installed * Tweak skip logic in test_accessors.py * flake8 * Address review comments * Temporarily remove cftime from py27 build environment on windows * flake8 * Install cftime via pip for Python 2.7 on Windows * flake8 * Remove unnecessary new lines; simplify _maybe_cast_to_cftimeindex * Restore test case for #2002 in test_coding_times.py I must have inadvertently removed it during a merge. * Tweak dates out of range warning logic slightly to preserve current default * Address review comments --- doc/time-series.rst | 96 ++- doc/whats-new.rst | 10 + xarray/coding/cftimeindex.py | 252 +++++++ xarray/coding/times.py | 127 ++-- xarray/core/accessors.py | 33 +- xarray/core/common.py | 31 + xarray/core/dataset.py | 5 +- xarray/core/options.py | 4 + xarray/core/utils.py | 18 +- xarray/plot/plot.py | 8 +- xarray/tests/test_accessors.py | 118 +++- xarray/tests/test_backends.py | 76 ++- xarray/tests/test_cftimeindex.py | 555 ++++++++++++++++ xarray/tests/test_coding_times.py | 1019 ++++++++++++++++++++--------- xarray/tests/test_dataarray.py | 19 +- xarray/tests/test_plot.py | 24 +- xarray/tests/test_utils.py | 67 +- 17 files changed, 2095 insertions(+), 367 deletions(-) create mode 100644 xarray/coding/cftimeindex.py create mode 100644 xarray/tests/test_cftimeindex.py diff --git a/doc/time-series.rst b/doc/time-series.rst index afd9f087bfe..5b857789629 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -70,7 +70,11 @@ You can manual decode arrays in this form by passing a dataset to One unfortunate limitation of using ``datetime64[ns]`` is that it limits the native representation of dates to those that fall between the years 1678 and 2262. When a netCDF file contains dates outside of these bounds, dates will be -returned as arrays of ``netcdftime.datetime`` objects. +returned as arrays of ``cftime.datetime`` objects and a ``CFTimeIndex`` +can be used for indexing. The ``CFTimeIndex`` enables only a subset of +the indexing functionality of a ``pandas.DatetimeIndex`` and is only enabled +when using standalone version of ``cftime`` (not the version packaged with +earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more information. Datetime indexing ----------------- @@ -207,3 +211,93 @@ Dataset and DataArray objects with an arbitrary number of dimensions. For more examples of using grouped operations on a time dimension, see :ref:`toy weather data`. + + +.. _CFTimeIndex: + +Non-standard calendars and dates outside the Timestamp-valid range +------------------------------------------------------------------ + +Through the standalone ``cftime`` library and a custom subclass of +``pandas.Index``, xarray supports a subset of the indexing functionality enabled +through the standard ``pandas.DatetimeIndex`` for dates from non-standard +calendars or dates using a standard calendar, but outside the +`Timestamp-valid range`_ (approximately between years 1678 and 2262). This +behavior has not yet been turned on by default; to take advantage of this +functionality, you must have the ``enable_cftimeindex`` option set to +``True`` within your context (see :py:func:`~xarray.set_options` for more +information). It is expected that this will become the default behavior in +xarray version 0.11. + +For instance, you can create a DataArray indexed by a time +coordinate with a no-leap calendar within a context manager setting the +``enable_cftimeindex`` option, and the time index will be cast to a +``CFTimeIndex``: + +.. ipython:: python + + from itertools import product + from cftime import DatetimeNoLeap + + dates = [DatetimeNoLeap(year, month, 1) for year, month in + product(range(1, 3), range(1, 13))] + with xr.set_options(enable_cftimeindex=True): + da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], + name='foo') + +.. note:: + + With the ``enable_cftimeindex`` option activated, a ``CFTimeIndex`` + will be used for time indexing if any of the following are true: + + - The dates are from a non-standard calendar + - Any dates are outside the Timestamp-valid range + + Otherwise a ``pandas.DatetimeIndex`` will be used. In addition, if any + variable (not just an index variable) is encoded using a non-standard + calendar, its times will be decoded into ``cftime.datetime`` objects, + regardless of whether or not they can be represented using + ``np.datetime64[ns]`` objects. + +For data indexed by a ``CFTimeIndex`` xarray currently supports: + +- `Partial datetime string indexing`_ using strictly `ISO 8601-format`_ partial + datetime strings: + +.. ipython:: python + + da.sel(time='0001') + da.sel(time=slice('0001-05', '0002-02')) + +- Access of basic datetime components via the ``dt`` accessor (in this case + just "year", "month", "day", "hour", "minute", "second", "microsecond", and + "season"): + +.. ipython:: python + + da.time.dt.year + da.time.dt.month + da.time.dt.season + +- Group-by operations based on datetime accessor attributes (e.g. by month of + the year): + +.. ipython:: python + + da.groupby('time.month').sum() + +- And serialization: + +.. ipython:: python + + da.to_netcdf('example.nc') + xr.open_dataset('example.nc') + +.. note:: + + Currently resampling along the time dimension for data indexed by a + ``CFTimeIndex`` is not supported. + +.. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timestamp-limitations +.. _ISO 8601-format: https://en.wikipedia.org/wiki/ISO_8601 +.. _partial datetime string indexing: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#partial-string-indexing diff --git a/doc/whats-new.rst b/doc/whats-new.rst index cc5506b553c..fc5f8bf3266 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -34,6 +34,16 @@ v0.10.4 (unreleased) Enhancements ~~~~~~~~~~~~ +- Add an option for using a ``CFTimeIndex`` for indexing times with + non-standard calendars and/or outside the Timestamp-valid range; this index + enables a subset of the functionality of a standard + ``pandas.DatetimeIndex`` (:issue:`789`, :issue:`1084`, :issue:`1252`). + By `Spencer Clark `_ with help from + `Stephan Hoyer `_. +- Allow for serialization of ``cftime.datetime`` objects (:issue:`789`, + :issue:`1084`, :issue:`2008`, :issue:`1252`) using the standalone ``cftime`` + library. By `Spencer Clark + `_. - Support writing lists of strings as netCDF attributes (:issue:`2044`). By `Dan Nowacki `_. - :py:meth:`~xarray.Dataset.to_netcdf(engine='h5netcdf')` now accepts h5py diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py new file mode 100644 index 00000000000..fb51ace5d69 --- /dev/null +++ b/xarray/coding/cftimeindex.py @@ -0,0 +1,252 @@ +from __future__ import absolute_import +import re +from datetime import timedelta + +import numpy as np +import pandas as pd + +from xarray.core import pycompat +from xarray.core.utils import is_scalar + + +def named(name, pattern): + return '(?P<' + name + '>' + pattern + ')' + + +def optional(x): + return '(?:' + x + ')?' + + +def trailing_optional(xs): + if not xs: + return '' + return xs[0] + optional(trailing_optional(xs[1:])) + + +def build_pattern(date_sep='\-', datetime_sep='T', time_sep='\:'): + pieces = [(None, 'year', '\d{4}'), + (date_sep, 'month', '\d{2}'), + (date_sep, 'day', '\d{2}'), + (datetime_sep, 'hour', '\d{2}'), + (time_sep, 'minute', '\d{2}'), + (time_sep, 'second', '\d{2}')] + pattern_list = [] + for sep, name, sub_pattern in pieces: + pattern_list.append((sep if sep else '') + named(name, sub_pattern)) + # TODO: allow timezone offsets? + return '^' + trailing_optional(pattern_list) + '$' + + +_BASIC_PATTERN = build_pattern(date_sep='', time_sep='') +_EXTENDED_PATTERN = build_pattern() +_PATTERNS = [_BASIC_PATTERN, _EXTENDED_PATTERN] + + +def parse_iso8601(datetime_string): + for pattern in _PATTERNS: + match = re.match(pattern, datetime_string) + if match: + return match.groupdict() + raise ValueError('no ISO-8601 match for string: %s' % datetime_string) + + +def _parse_iso8601_with_reso(date_type, timestr): + default = date_type(1, 1, 1) + result = parse_iso8601(timestr) + replace = {} + + for attr in ['year', 'month', 'day', 'hour', 'minute', 'second']: + value = result.get(attr, None) + if value is not None: + # Note ISO8601 conventions allow for fractional seconds. + # TODO: Consider adding support for sub-second resolution? + replace[attr] = int(value) + resolution = attr + + return default.replace(**replace), resolution + + +def _parsed_string_to_bounds(date_type, resolution, parsed): + """Generalization of + pandas.tseries.index.DatetimeIndex._parsed_string_to_bounds + for use with non-standard calendars and cftime.datetime + objects. + """ + if resolution == 'year': + return (date_type(parsed.year, 1, 1), + date_type(parsed.year + 1, 1, 1) - timedelta(microseconds=1)) + elif resolution == 'month': + if parsed.month == 12: + end = date_type(parsed.year + 1, 1, 1) - timedelta(microseconds=1) + else: + end = (date_type(parsed.year, parsed.month + 1, 1) - + timedelta(microseconds=1)) + return date_type(parsed.year, parsed.month, 1), end + elif resolution == 'day': + start = date_type(parsed.year, parsed.month, parsed.day) + return start, start + timedelta(days=1, microseconds=-1) + elif resolution == 'hour': + start = date_type(parsed.year, parsed.month, parsed.day, parsed.hour) + return start, start + timedelta(hours=1, microseconds=-1) + elif resolution == 'minute': + start = date_type(parsed.year, parsed.month, parsed.day, parsed.hour, + parsed.minute) + return start, start + timedelta(minutes=1, microseconds=-1) + elif resolution == 'second': + start = date_type(parsed.year, parsed.month, parsed.day, parsed.hour, + parsed.minute, parsed.second) + return start, start + timedelta(seconds=1, microseconds=-1) + else: + raise KeyError + + +def get_date_field(datetimes, field): + """Adapted from pandas.tslib.get_date_field""" + return np.array([getattr(date, field) for date in datetimes]) + + +def _field_accessor(name, docstring=None): + """Adapted from pandas.tseries.index._field_accessor""" + def f(self): + return get_date_field(self._data, name) + + f.__name__ = name + f.__doc__ = docstring + return property(f) + + +def get_date_type(self): + return type(self._data[0]) + + +def assert_all_valid_date_type(data): + import cftime + + sample = data[0] + date_type = type(sample) + if not isinstance(sample, cftime.datetime): + raise TypeError( + 'CFTimeIndex requires cftime.datetime ' + 'objects. Got object of {}.'.format(date_type)) + if not all(isinstance(value, date_type) for value in data): + raise TypeError( + 'CFTimeIndex requires using datetime ' + 'objects of all the same type. Got\n{}.'.format(data)) + + +class CFTimeIndex(pd.Index): + year = _field_accessor('year', 'The year of the datetime') + month = _field_accessor('month', 'The month of the datetime') + day = _field_accessor('day', 'The days of the datetime') + hour = _field_accessor('hour', 'The hours of the datetime') + minute = _field_accessor('minute', 'The minutes of the datetime') + second = _field_accessor('second', 'The seconds of the datetime') + microsecond = _field_accessor('microsecond', + 'The microseconds of the datetime') + date_type = property(get_date_type) + + def __new__(cls, data): + result = object.__new__(cls) + assert_all_valid_date_type(data) + result._data = np.array(data) + return result + + def _partial_date_slice(self, resolution, parsed): + """Adapted from + pandas.tseries.index.DatetimeIndex._partial_date_slice + + Note that when using a CFTimeIndex, if a partial-date selection + returns a single element, it will never be converted to a scalar + coordinate; this is in slight contrast to the behavior when using + a DatetimeIndex, which sometimes will return a DataArray with a scalar + coordinate depending on the resolution of the datetimes used in + defining the index. For example: + + >>> from cftime import DatetimeNoLeap + >>> import pandas as pd + >>> import xarray as xr + >>> da = xr.DataArray([1, 2], + coords=[[DatetimeNoLeap(2001, 1, 1), + DatetimeNoLeap(2001, 2, 1)]], + dims=['time']) + >>> da.sel(time='2001-01-01') + + array([1]) + Coordinates: + * time (time) object 2001-01-01 00:00:00 + >>> da = xr.DataArray([1, 2], + coords=[[pd.Timestamp(2001, 1, 1), + pd.Timestamp(2001, 2, 1)]], + dims=['time']) + >>> da.sel(time='2001-01-01') + + array(1) + Coordinates: + time datetime64[ns] 2001-01-01 + >>> da = xr.DataArray([1, 2], + coords=[[pd.Timestamp(2001, 1, 1, 1), + pd.Timestamp(2001, 2, 1)]], + dims=['time']) + >>> da.sel(time='2001-01-01') + + array([1]) + Coordinates: + * time (time) datetime64[ns] 2001-01-01T01:00:00 + """ + start, end = _parsed_string_to_bounds(self.date_type, resolution, + parsed) + lhs_mask = (self._data >= start) + rhs_mask = (self._data <= end) + return (lhs_mask & rhs_mask).nonzero()[0] + + def _get_string_slice(self, key): + """Adapted from pandas.tseries.index.DatetimeIndex._get_string_slice""" + parsed, resolution = _parse_iso8601_with_reso(self.date_type, key) + loc = self._partial_date_slice(resolution, parsed) + return loc + + def get_loc(self, key, method=None, tolerance=None): + """Adapted from pandas.tseries.index.DatetimeIndex.get_loc""" + if isinstance(key, pycompat.basestring): + return self._get_string_slice(key) + else: + return pd.Index.get_loc(self, key, method=method, + tolerance=tolerance) + + def _maybe_cast_slice_bound(self, label, side, kind): + """Adapted from + pandas.tseries.index.DatetimeIndex._maybe_cast_slice_bound""" + if isinstance(label, pycompat.basestring): + parsed, resolution = _parse_iso8601_with_reso(self.date_type, + label) + start, end = _parsed_string_to_bounds(self.date_type, resolution, + parsed) + if self.is_monotonic_decreasing and len(self): + return end if side == 'left' else start + return start if side == 'left' else end + else: + return label + + # TODO: Add ability to use integer range outside of iloc? + # e.g. series[1:5]. + def get_value(self, series, key): + """Adapted from pandas.tseries.index.DatetimeIndex.get_value""" + if not isinstance(key, slice): + return series.iloc[self.get_loc(key)] + else: + return series.iloc[self.slice_indexer( + key.start, key.stop, key.step)] + + def __contains__(self, key): + """Adapted from + pandas.tseries.base.DatetimeIndexOpsMixin.__contains__""" + try: + result = self.get_loc(key) + return (is_scalar(result) or type(result) == slice or + (isinstance(result, np.ndarray) and result.size)) + except (KeyError, TypeError, ValueError): + return False + + def contains(self, key): + """Needed for .loc based partial-string indexing""" + return self.__contains__(key) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 0a48b62986e..61314d9cbe6 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -9,8 +9,10 @@ import numpy as np import pandas as pd +from ..core.common import contains_cftime_datetimes from ..core import indexing from ..core.formatting import first_n_items, format_timestamp, last_item +from ..core.options import OPTIONS from ..core.pycompat import PY3 from ..core.variable import Variable from .variables import ( @@ -24,7 +26,7 @@ from pandas.tslib import OutOfBoundsDatetime -# standard calendars recognized by netcdftime +# standard calendars recognized by cftime _STANDARD_CALENDARS = set(['standard', 'gregorian', 'proleptic_gregorian']) _NS_PER_TIME_DELTA = {'us': int(1e3), @@ -54,6 +56,15 @@ def _import_cftime(): return cftime +def _require_standalone_cftime(): + """Raises an ImportError if the standalone cftime is not found""" + try: + import cftime # noqa: F401 + except ImportError: + raise ImportError('Using a CFTimeIndex requires the standalone ' + 'version of the cftime library.') + + def _netcdf_to_numpy_timeunit(units): units = units.lower() if not units.endswith('s'): @@ -73,28 +84,41 @@ def _unpack_netcdf_time_units(units): return delta_units, ref_date -def _decode_datetime_with_netcdftime(num_dates, units, calendar): +def _decode_datetime_with_cftime(num_dates, units, calendar, + enable_cftimeindex): cftime = _import_cftime() + if enable_cftimeindex: + _require_standalone_cftime() + dates = np.asarray(cftime.num2date(num_dates, units, calendar, + only_use_cftime_datetimes=True)) + else: + dates = np.asarray(cftime.num2date(num_dates, units, calendar)) - dates = np.asarray(cftime.num2date(num_dates, units, calendar)) if (dates[np.nanargmin(num_dates)].year < 1678 or dates[np.nanargmax(num_dates)].year >= 2262): - warnings.warn('Unable to decode time axis into full ' - 'numpy.datetime64 objects, continuing using dummy ' - 'netcdftime.datetime objects instead, reason: dates out' - ' of range', SerializationWarning, stacklevel=3) + if not enable_cftimeindex or calendar in _STANDARD_CALENDARS: + warnings.warn( + 'Unable to decode time axis into full ' + 'numpy.datetime64 objects, continuing using dummy ' + 'cftime.datetime objects instead, reason: dates out ' + 'of range', SerializationWarning, stacklevel=3) else: - try: - dates = cftime_to_nptime(dates) - except ValueError as e: - warnings.warn('Unable to decode time axis into full ' - 'numpy.datetime64 objects, continuing using ' - 'dummy netcdftime.datetime objects instead, reason:' - '{0}'.format(e), SerializationWarning, stacklevel=3) + if enable_cftimeindex: + if calendar in _STANDARD_CALENDARS: + dates = cftime_to_nptime(dates) + else: + try: + dates = cftime_to_nptime(dates) + except ValueError as e: + warnings.warn( + 'Unable to decode time axis into full ' + 'numpy.datetime64 objects, continuing using ' + 'dummy cftime.datetime objects instead, reason:' + '{0}'.format(e), SerializationWarning, stacklevel=3) return dates -def _decode_cf_datetime_dtype(data, units, calendar): +def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): # Verify that at least the first and last date can be decoded # successfully. Otherwise, tracebacks end up swallowed by # Dataset.__repr__ when users try to view their lazily decoded array. @@ -104,7 +128,8 @@ def _decode_cf_datetime_dtype(data, units, calendar): last_item(values) or [0]]) try: - result = decode_cf_datetime(example_value, units, calendar) + result = decode_cf_datetime(example_value, units, calendar, + enable_cftimeindex) except Exception: calendar_msg = ('the default calendar' if calendar is None else 'calendar %r' % calendar) @@ -120,12 +145,13 @@ def _decode_cf_datetime_dtype(data, units, calendar): return dtype -def decode_cf_datetime(num_dates, units, calendar=None): +def decode_cf_datetime(num_dates, units, calendar=None, + enable_cftimeindex=False): """Given an array of numeric dates in netCDF format, convert it into a numpy array of date time objects. For standard (Gregorian) calendars, this function uses vectorized - operations, which makes it much faster than netcdftime.num2date. In such a + operations, which makes it much faster than cftime.num2date. In such a case, the returned array will be of type np.datetime64. Note that time unit in `units` must not be smaller than microseconds and @@ -133,7 +159,7 @@ def decode_cf_datetime(num_dates, units, calendar=None): See also -------- - netcdftime.num2date + cftime.num2date """ num_dates = np.asarray(num_dates) flat_num_dates = num_dates.ravel() @@ -151,7 +177,7 @@ def decode_cf_datetime(num_dates, units, calendar=None): ref_date = pd.Timestamp(ref_date) except ValueError: # ValueError is raised by pd.Timestamp for non-ISO timestamp - # strings, in which case we fall back to using netcdftime + # strings, in which case we fall back to using cftime raise OutOfBoundsDatetime # fixes: https://github.com/pydata/pandas/issues/14068 @@ -170,8 +196,9 @@ def decode_cf_datetime(num_dates, units, calendar=None): ref_date).values except (OutOfBoundsDatetime, OverflowError): - dates = _decode_datetime_with_netcdftime( - flat_num_dates.astype(np.float), units, calendar) + dates = _decode_datetime_with_cftime( + flat_num_dates.astype(np.float), units, calendar, + enable_cftimeindex) return dates.reshape(num_dates.shape) @@ -203,18 +230,41 @@ def _infer_time_units_from_diff(unique_timedeltas): return 'seconds' +def infer_calendar_name(dates): + """Given an array of datetimes, infer the CF calendar name""" + if np.asarray(dates).dtype == 'datetime64[ns]': + return 'proleptic_gregorian' + else: + return np.asarray(dates).ravel()[0].calendar + + def infer_datetime_units(dates): """Given an array of datetimes, returns a CF compatible time-unit string of the form "{time_unit} since {date[0]}", where `time_unit` is 'days', 'hours', 'minutes' or 'seconds' (the first one that can evenly divide all unique time deltas in `dates`) """ - dates = pd.to_datetime(np.asarray(dates).ravel(), box=False) - dates = dates[pd.notnull(dates)] - unique_timedeltas = np.unique(np.diff(dates)) + dates = np.asarray(dates).ravel() + if np.asarray(dates).dtype == 'datetime64[ns]': + dates = pd.to_datetime(dates, box=False) + dates = dates[pd.notnull(dates)] + reference_date = dates[0] if len(dates) > 0 else '1970-01-01' + reference_date = pd.Timestamp(reference_date) + else: + reference_date = dates[0] if len(dates) > 0 else '1970-01-01' + reference_date = format_cftime_datetime(reference_date) + unique_timedeltas = np.unique(np.diff(dates)).astype('timedelta64[ns]') units = _infer_time_units_from_diff(unique_timedeltas) - reference_date = dates[0] if len(dates) > 0 else '1970-01-01' - return '%s since %s' % (units, pd.Timestamp(reference_date)) + return '%s since %s' % (units, reference_date) + + +def format_cftime_datetime(date): + """Converts a cftime.datetime object to a string with the format: + YYYY-MM-DD HH:MM:SS.UUUUUU + """ + return '{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}.{:06d}'.format( + date.year, date.month, date.day, date.hour, date.minute, date.second, + date.microsecond) def infer_timedelta_units(deltas): @@ -249,8 +299,8 @@ def _cleanup_netcdf_time_units(units): return units -def _encode_datetime_with_netcdftime(dates, units, calendar): - """Fallback method for encoding dates using netcdftime. +def _encode_datetime_with_cftime(dates, units, calendar): + """Fallback method for encoding dates using cftime. This method is more flexible than xarray's parsing using datetime64[ns] arrays but also slower because it loops over each element. @@ -282,7 +332,7 @@ def encode_cf_datetime(dates, units=None, calendar=None): See also -------- - netcdftime.date2num + cftime.date2num """ dates = np.asarray(dates) @@ -292,12 +342,12 @@ def encode_cf_datetime(dates, units=None, calendar=None): units = _cleanup_netcdf_time_units(units) if calendar is None: - calendar = 'proleptic_gregorian' + calendar = infer_calendar_name(dates) delta, ref_date = _unpack_netcdf_time_units(units) try: if calendar not in _STANDARD_CALENDARS or dates.dtype.kind == 'O': - # parse with netcdftime instead + # parse with cftime instead raise OutOfBoundsDatetime assert dates.dtype == 'datetime64[ns]' @@ -307,7 +357,7 @@ def encode_cf_datetime(dates, units=None, calendar=None): num = (dates - ref_date) / time_delta except (OutOfBoundsDatetime, OverflowError): - num = _encode_datetime_with_netcdftime(dates, units, calendar) + num = _encode_datetime_with_cftime(dates, units, calendar) num = cast_to_int_if_safe(num) return (num, units, calendar) @@ -328,8 +378,8 @@ class CFDatetimeCoder(VariableCoder): def encode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_encoding(variable) - - if np.issubdtype(data.dtype, np.datetime64): + if (np.issubdtype(data.dtype, np.datetime64) or + contains_cftime_datetimes(variable)): (data, units, calendar) = encode_cf_datetime( data, encoding.pop('units', None), @@ -342,12 +392,15 @@ def encode(self, variable, name=None): def decode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_decoding(variable) + enable_cftimeindex = OPTIONS['enable_cftimeindex'] if 'units' in attrs and 'since' in attrs['units']: units = pop_to(attrs, encoding, 'units') calendar = pop_to(attrs, encoding, 'calendar') - dtype = _decode_cf_datetime_dtype(data, units, calendar) + dtype = _decode_cf_datetime_dtype( + data, units, calendar, enable_cftimeindex) transform = partial( - decode_cf_datetime, units=units, calendar=calendar) + decode_cf_datetime, units=units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) data = lazy_elemwise_func(data, transform, dtype) return Variable(dims, data, attrs, encoding) diff --git a/xarray/core/accessors.py b/xarray/core/accessors.py index 52d9e6db408..81af0532d93 100644 --- a/xarray/core/accessors.py +++ b/xarray/core/accessors.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from .dtypes import is_datetime_like +from .common import is_np_datetime_like, _contains_datetime_like_objects from .pycompat import dask_array_type @@ -16,6 +16,20 @@ def _season_from_months(months): return seasons[(months // 3) % 4] +def _access_through_cftimeindex(values, name): + """Coerce an array of datetime-like values to a CFTimeIndex + and access requested datetime component + """ + from ..coding.cftimeindex import CFTimeIndex + values_as_cftimeindex = CFTimeIndex(values.ravel()) + if name == 'season': + months = values_as_cftimeindex.month + field_values = _season_from_months(months) + else: + field_values = getattr(values_as_cftimeindex, name) + return field_values.reshape(values.shape) + + def _access_through_series(values, name): """Coerce an array of datetime-like values to a pandas Series and access requested datetime component @@ -48,12 +62,17 @@ def _get_date_field(values, name, dtype): Array-like of datetime fields accessed for each element in values """ + if is_np_datetime_like(values.dtype): + access_method = _access_through_series + else: + access_method = _access_through_cftimeindex + if isinstance(values, dask_array_type): from dask.array import map_blocks - return map_blocks(_access_through_series, + return map_blocks(access_method, values, name, dtype=dtype) else: - return _access_through_series(values, name) + return access_method(values, name) def _round_series(values, name, freq): @@ -111,15 +130,17 @@ class DatetimeAccessor(object): All of the pandas fields are accessible here. Note that these fields are not calendar-aware; if your datetimes are encoded with a non-Gregorian - calendar (e.g. a 360-day calendar) using netcdftime, then some fields like + calendar (e.g. a 360-day calendar) using cftime, then some fields like `dayofyear` may not be accurate. """ def __init__(self, xarray_obj): - if not is_datetime_like(xarray_obj.dtype): + if not _contains_datetime_like_objects(xarray_obj): raise TypeError("'dt' accessor only available for " - "DataArray with datetime64 or timedelta64 dtype") + "DataArray with datetime64 timedelta64 dtype or " + "for arrays containing cftime datetime " + "objects.") self._obj = xarray_obj def _tslib_field_accessor(name, docstring=None, dtype=None): diff --git a/xarray/core/common.py b/xarray/core/common.py index 0c6e0fccd8e..f623091ebdb 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -933,3 +933,34 @@ def ones_like(other, dtype=None): """Shorthand for full_like(other, 1, dtype) """ return full_like(other, 1, dtype) + + +def is_np_datetime_like(dtype): + """Check if a dtype is a subclass of the numpy datetime types + """ + return (np.issubdtype(dtype, np.datetime64) or + np.issubdtype(dtype, np.timedelta64)) + + +def contains_cftime_datetimes(var): + """Check if a variable contains cftime datetime objects""" + try: + from cftime import datetime as cftime_datetime + except ImportError: + return False + else: + if var.dtype == np.dtype('O') and var.data.size > 0: + sample = var.data.ravel()[0] + if isinstance(sample, dask_array_type): + sample = sample.compute() + if isinstance(sample, np.ndarray): + sample = sample.item() + return isinstance(sample, cftime_datetime) + else: + return False + + +def _contains_datetime_like_objects(var): + """Check if a variable contains datetime like objects (either + np.datetime64, np.timedelta64, or cftime.datetime)""" + return is_np_datetime_like(var.dtype) or contains_cftime_datetimes(var) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 32913127636..bdb2bf86990 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -17,7 +17,8 @@ rolling, utils) from .. import conventions from .alignment import align -from .common import DataWithCoords, ImplementsDatasetReduce +from .common import (DataWithCoords, ImplementsDatasetReduce, + _contains_datetime_like_objects) from .coordinates import ( DatasetCoordinates, Indexes, LevelCoordinatesSource, assert_coordinate_consistent, remap_label_indexers) @@ -75,7 +76,7 @@ def _get_virtual_variable(variables, key, level_vars=None, dim_sizes=None): virtual_var = ref_var var_name = key else: - if is_datetime_like(ref_var.dtype): + if _contains_datetime_like_objects(ref_var): ref_var = xr.DataArray(ref_var) data = getattr(ref_var.dt, var_name).data else: diff --git a/xarray/core/options.py b/xarray/core/options.py index b2968a2a02f..48d4567fc99 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -3,6 +3,7 @@ OPTIONS = { 'display_width': 80, 'arithmetic_join': 'inner', + 'enable_cftimeindex': False } @@ -15,6 +16,9 @@ class set_options(object): Default: ``80``. - ``arithmetic_join``: DataArray/Dataset alignment in binary operations. Default: ``'inner'``. + - ``enable_cftimeindex``: flag to enable using a ``CFTimeIndex`` + for time indexes with non-standard calendars or dates outside the + Timestamp-valid range. Default: ``False``. You can use ``set_options`` either as a context manager: diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 25a60b87266..06bb3ede393 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -12,6 +12,7 @@ import numpy as np import pandas as pd +from .options import OPTIONS from .pycompat import ( OrderedDict, basestring, bytes_type, dask_array_type, iteritems) @@ -36,6 +37,21 @@ def wrapper(*args, **kwargs): return wrapper +def _maybe_cast_to_cftimeindex(index): + from ..coding.cftimeindex import CFTimeIndex + + if not OPTIONS['enable_cftimeindex']: + return index + else: + if index.dtype == 'O': + try: + return CFTimeIndex(index) + except (ImportError, TypeError): + return index + else: + return index + + def safe_cast_to_index(array): """Given an array, safely cast it to a pandas.Index. @@ -54,7 +70,7 @@ def safe_cast_to_index(array): if hasattr(array, 'dtype') and array.dtype.kind == 'O': kwargs['dtype'] = object index = pd.Index(np.asarray(array), **kwargs) - return index + return _maybe_cast_to_cftimeindex(index) def multiindex_from_product_levels(levels, names=None): diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 6a3bed08f72..ee1df611d3b 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -14,6 +14,7 @@ import numpy as np import pandas as pd +from xarray.core.common import contains_cftime_datetimes from xarray.core.pycompat import basestring from .facetgrid import FacetGrid @@ -53,7 +54,8 @@ def _ensure_plottable(*args): if not (_valid_numpy_subdtype(np.array(x), numpy_types) or _valid_other_type(np.array(x), other_types)): raise TypeError('Plotting requires coordinates to be numeric ' - 'or dates.') + 'or dates of type np.datetime64 or ' + 'datetime.datetime.') def _easy_facetgrid(darray, plotfunc, x, y, row=None, col=None, @@ -120,6 +122,10 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, rtol=0.01, """ darray = darray.squeeze() + if contains_cftime_datetimes(darray): + raise NotImplementedError('Plotting arrays of cftime.datetime objects ' + 'is currently not possible.') + plot_dims = set(darray.dims) plot_dims.discard(row) plot_dims.discard(col) diff --git a/xarray/tests/test_accessors.py b/xarray/tests/test_accessors.py index ad521546d2e..e1b3a95b942 100644 --- a/xarray/tests/test_accessors.py +++ b/xarray/tests/test_accessors.py @@ -2,11 +2,13 @@ import numpy as np import pandas as pd +import pytest import xarray as xr from . import ( - TestCase, assert_array_equal, assert_equal, raises_regex, requires_dask) + TestCase, assert_array_equal, assert_equal, raises_regex, requires_dask, + has_cftime, has_dask, has_cftime_or_netCDF4) class TestDatetimeAccessor(TestCase): @@ -114,3 +116,117 @@ def test_rounders(self): xdates.time.dt.ceil('D').values) assert_array_equal(dates.round('D').values, xdates.time.dt.round('D').values) + + +_CFTIME_CALENDARS = ['365_day', '360_day', 'julian', 'all_leap', + '366_day', 'gregorian', 'proleptic_gregorian'] +_NT = 100 + + +@pytest.fixture(params=_CFTIME_CALENDARS) +def calendar(request): + return request.param + + +@pytest.fixture() +def times(calendar): + import cftime + + return cftime.num2date( + np.arange(_NT), units='hours since 2000-01-01', calendar=calendar, + only_use_cftime_datetimes=True) + + +@pytest.fixture() +def data(times): + data = np.random.rand(10, 10, _NT) + lons = np.linspace(0, 11, 10) + lats = np.linspace(0, 20, 10) + return xr.DataArray(data, coords=[lons, lats, times], + dims=['lon', 'lat', 'time'], name='data') + + +@pytest.fixture() +def times_3d(times): + lons = np.linspace(0, 11, 10) + lats = np.linspace(0, 20, 10) + times_arr = np.random.choice(times, size=(10, 10, _NT)) + return xr.DataArray(times_arr, coords=[lons, lats, times], + dims=['lon', 'lat', 'time'], + name='data') + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('field', ['year', 'month', 'day', 'hour']) +def test_field_access(data, field): + result = getattr(data.time.dt, field) + expected = xr.DataArray( + getattr(xr.coding.cftimeindex.CFTimeIndex(data.time.values), field), + name=field, coords=data.time.coords, dims=data.time.dims) + + assert_equal(result, expected) + + +@pytest.mark.skipif(not has_dask, reason='dask not installed') +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('field', ['year', 'month', 'day', 'hour']) +def test_dask_field_access_1d(data, field): + import dask.array as da + + expected = xr.DataArray( + getattr(xr.coding.cftimeindex.CFTimeIndex(data.time.values), field), + name=field, dims=['time']) + times = xr.DataArray(data.time.values, dims=['time']).chunk({'time': 50}) + result = getattr(times.dt, field) + assert isinstance(result.data, da.Array) + assert result.chunks == times.chunks + assert_equal(result.compute(), expected) + + +@pytest.mark.skipif(not has_dask, reason='dask not installed') +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('field', ['year', 'month', 'day', 'hour']) +def test_dask_field_access(times_3d, data, field): + import dask.array as da + + expected = xr.DataArray( + getattr(xr.coding.cftimeindex.CFTimeIndex(times_3d.values.ravel()), + field).reshape(times_3d.shape), + name=field, coords=times_3d.coords, dims=times_3d.dims) + times_3d = times_3d.chunk({'lon': 5, 'lat': 5, 'time': 50}) + result = getattr(times_3d.dt, field) + assert isinstance(result.data, da.Array) + assert result.chunks == times_3d.chunks + assert_equal(result.compute(), expected) + + +@pytest.fixture() +def cftime_date_type(calendar): + from .test_coding_times import _all_cftime_date_types + + return _all_cftime_date_types()[calendar] + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_seasons(cftime_date_type): + dates = np.array([cftime_date_type(2000, month, 15) + for month in range(1, 13)]) + dates = xr.DataArray(dates) + seasons = ['DJF', 'DJF', 'MAM', 'MAM', 'MAM', 'JJA', + 'JJA', 'JJA', 'SON', 'SON', 'SON', 'DJF'] + seasons = xr.DataArray(seasons) + + assert_array_equal(seasons.values, dates.dt.season.values) + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, + reason='cftime or netCDF4 not installed') +def test_dt_accessor_error_netCDF4(cftime_date_type): + da = xr.DataArray( + [cftime_date_type(1, 1, 1), cftime_date_type(2, 1, 1)], + dims=['time']) + if not has_cftime: + with pytest.raises(TypeError): + da.dt.month + else: + da.dt.month diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 632145007e2..2d4e5c0f261 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -32,7 +32,8 @@ assert_identical, has_dask, has_netCDF4, has_scipy, network, raises_regex, requires_dask, requires_h5netcdf, requires_netCDF4, requires_pathlib, requires_pydap, requires_pynio, requires_rasterio, requires_scipy, - requires_scipy_or_netCDF4, requires_zarr) + requires_scipy_or_netCDF4, requires_zarr, + requires_cftime) from .test_dataset import create_test_data try: @@ -341,7 +342,7 @@ def test_roundtrip_string_encoded_characters(self): assert_identical(expected, actual) self.assertEqual(actual['x'].encoding['_Encoding'], 'ascii') - def test_roundtrip_datetime_data(self): + def test_roundtrip_numpy_datetime_data(self): times = pd.to_datetime(['2000-01-01', '2000-01-02', 'NaT']) expected = Dataset({'t': ('t', times), 't0': times[0]}) kwds = {'encoding': {'t0': {'units': 'days since 1950-01-01'}}} @@ -349,6 +350,35 @@ def test_roundtrip_datetime_data(self): assert_identical(expected, actual) assert actual.t0.encoding['units'] == 'days since 1950-01-01' + @requires_cftime + def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + from .test_coding_times import _all_cftime_date_types + + date_types = _all_cftime_date_types() + for date_type in date_types.values(): + times = [date_type(1, 1, 1), date_type(1, 1, 2)] + expected = Dataset({'t': ('t', times), 't0': times[0]}) + kwds = {'encoding': {'t0': {'units': 'days since 0001-01-01'}}} + expected_decoded_t = np.array(times) + expected_decoded_t0 = np.array([date_type(1, 1, 1)]) + expected_calendar = times[0].calendar + + with xr.set_options(enable_cftimeindex=True): + with self.roundtrip(expected, save_kwargs=kwds) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t.encoding['units'] == + 'days since 0001-01-01 00:00:00.000000') + assert (actual.t.encoding['calendar'] == + expected_calendar) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t0.encoding['units'] == + 'days since 0001-01-01') + assert (actual.t.encoding['calendar'] == + expected_calendar) + def test_roundtrip_timedelta_data(self): time_deltas = pd.to_timedelta(['1h', '2h', 'NaT']) expected = Dataset({'td': ('td', time_deltas), 'td0': time_deltas[0]}) @@ -1949,7 +1979,7 @@ def test_roundtrip_string_encoded_characters(self): def test_roundtrip_coordinates_with_space(self): pass - def test_roundtrip_datetime_data(self): + def test_roundtrip_numpy_datetime_data(self): # Override method in DatasetIOTestCases - remove not applicable # save_kwds times = pd.to_datetime(['2000-01-01', '2000-01-02', 'NaT']) @@ -1957,6 +1987,46 @@ def test_roundtrip_datetime_data(self): with self.roundtrip(expected) as actual: assert_identical(expected, actual) + def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + # Override method in DatasetIOTestCases - remove not applicable + # save_kwds + from .test_coding_times import _all_cftime_date_types + + date_types = _all_cftime_date_types() + for date_type in date_types.values(): + times = [date_type(1, 1, 1), date_type(1, 1, 2)] + expected = Dataset({'t': ('t', times), 't0': times[0]}) + expected_decoded_t = np.array(times) + expected_decoded_t0 = np.array([date_type(1, 1, 1)]) + + with xr.set_options(enable_cftimeindex=True): + with self.roundtrip(expected) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + + def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): + # Override method in DatasetIOTestCases - remove not applicable + # save_kwds + from .test_coding_times import _all_cftime_date_types + + date_types = _all_cftime_date_types() + for date_type in date_types.values(): + times = [date_type(1, 1, 1), date_type(1, 1, 2)] + expected = Dataset({'t': ('t', times), 't0': times[0]}) + expected_decoded_t = np.array(times) + expected_decoded_t0 = np.array([date_type(1, 1, 1)]) + + with xr.set_options(enable_cftimeindex=False): + with self.roundtrip(expected) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + self.assertTrue((abs_diff <= np.timedelta64(1, 's')).all()) + def test_write_store(self): # Override method in DatasetIOTestCases - not applicable to dask pass diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py new file mode 100644 index 00000000000..c78ac038bd5 --- /dev/null +++ b/xarray/tests/test_cftimeindex.py @@ -0,0 +1,555 @@ +from __future__ import absolute_import + +import pytest + +import pandas as pd +import xarray as xr + +from datetime import timedelta +from xarray.coding.cftimeindex import ( + parse_iso8601, CFTimeIndex, assert_all_valid_date_type, + _parsed_string_to_bounds, _parse_iso8601_with_reso) +from xarray.tests import assert_array_equal, assert_identical + +from . import has_cftime, has_cftime_or_netCDF4 +from .test_coding_times import _all_cftime_date_types + + +def date_dict(year=None, month=None, day=None, + hour=None, minute=None, second=None): + return dict(year=year, month=month, day=day, hour=hour, + minute=minute, second=second) + + +ISO8601_STRING_TESTS = { + 'year': ('1999', date_dict(year='1999')), + 'month': ('199901', date_dict(year='1999', month='01')), + 'month-dash': ('1999-01', date_dict(year='1999', month='01')), + 'day': ('19990101', date_dict(year='1999', month='01', day='01')), + 'day-dash': ('1999-01-01', date_dict(year='1999', month='01', day='01')), + 'hour': ('19990101T12', date_dict( + year='1999', month='01', day='01', hour='12')), + 'hour-dash': ('1999-01-01T12', date_dict( + year='1999', month='01', day='01', hour='12')), + 'minute': ('19990101T1234', date_dict( + year='1999', month='01', day='01', hour='12', minute='34')), + 'minute-dash': ('1999-01-01T12:34', date_dict( + year='1999', month='01', day='01', hour='12', minute='34')), + 'second': ('19990101T123456', date_dict( + year='1999', month='01', day='01', hour='12', minute='34', + second='56')), + 'second-dash': ('1999-01-01T12:34:56', date_dict( + year='1999', month='01', day='01', hour='12', minute='34', + second='56')) +} + + +@pytest.mark.parametrize(('string', 'expected'), + list(ISO8601_STRING_TESTS.values()), + ids=list(ISO8601_STRING_TESTS.keys())) +def test_parse_iso8601(string, expected): + result = parse_iso8601(string) + assert result == expected + + with pytest.raises(ValueError): + parse_iso8601(string + '3') + parse_iso8601(string + '.3') + + +_CFTIME_CALENDARS = ['365_day', '360_day', 'julian', 'all_leap', + '366_day', 'gregorian', 'proleptic_gregorian'] + + +@pytest.fixture(params=_CFTIME_CALENDARS) +def date_type(request): + return _all_cftime_date_types()[request.param] + + +@pytest.fixture +def index(date_type): + dates = [date_type(1, 1, 1), date_type(1, 2, 1), + date_type(2, 1, 1), date_type(2, 2, 1)] + return CFTimeIndex(dates) + + +@pytest.fixture +def monotonic_decreasing_index(date_type): + dates = [date_type(2, 2, 1), date_type(2, 1, 1), + date_type(1, 2, 1), date_type(1, 1, 1)] + return CFTimeIndex(dates) + + +@pytest.fixture +def da(index): + return xr.DataArray([1, 2, 3, 4], coords=[index], + dims=['time']) + + +@pytest.fixture +def series(index): + return pd.Series([1, 2, 3, 4], index=index) + + +@pytest.fixture +def df(index): + return pd.DataFrame([1, 2, 3, 4], index=index) + + +@pytest.fixture +def feb_days(date_type): + import cftime + if date_type is cftime.DatetimeAllLeap: + return 29 + elif date_type is cftime.Datetime360Day: + return 30 + else: + return 28 + + +@pytest.fixture +def dec_days(date_type): + import cftime + if date_type is cftime.Datetime360Day: + return 30 + else: + return 31 + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_assert_all_valid_date_type(date_type, index): + import cftime + if date_type is cftime.DatetimeNoLeap: + mixed_date_types = [date_type(1, 1, 1), + cftime.DatetimeAllLeap(1, 2, 1)] + else: + mixed_date_types = [date_type(1, 1, 1), + cftime.DatetimeNoLeap(1, 2, 1)] + with pytest.raises(TypeError): + assert_all_valid_date_type(mixed_date_types) + + with pytest.raises(TypeError): + assert_all_valid_date_type([1, date_type(1, 1, 1)]) + + assert_all_valid_date_type([date_type(1, 1, 1), date_type(1, 2, 1)]) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize(('field', 'expected'), [ + ('year', [1, 1, 2, 2]), + ('month', [1, 2, 1, 2]), + ('day', [1, 1, 1, 1]), + ('hour', [0, 0, 0, 0]), + ('minute', [0, 0, 0, 0]), + ('second', [0, 0, 0, 0]), + ('microsecond', [0, 0, 0, 0])]) +def test_cftimeindex_field_accessors(index, field, expected): + result = getattr(index, field) + assert_array_equal(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize(('string', 'date_args', 'reso'), [ + ('1999', (1999, 1, 1), 'year'), + ('199902', (1999, 2, 1), 'month'), + ('19990202', (1999, 2, 2), 'day'), + ('19990202T01', (1999, 2, 2, 1), 'hour'), + ('19990202T0101', (1999, 2, 2, 1, 1), 'minute'), + ('19990202T010156', (1999, 2, 2, 1, 1, 56), 'second')]) +def test_parse_iso8601_with_reso(date_type, string, date_args, reso): + expected_date = date_type(*date_args) + expected_reso = reso + result_date, result_reso = _parse_iso8601_with_reso(date_type, string) + assert result_date == expected_date + assert result_reso == expected_reso + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_parse_string_to_bounds_year(date_type, dec_days): + parsed = date_type(2, 2, 10, 6, 2, 8, 1) + expected_start = date_type(2, 1, 1) + expected_end = date_type(2, 12, dec_days, 23, 59, 59, 999999) + result_start, result_end = _parsed_string_to_bounds( + date_type, 'year', parsed) + assert result_start == expected_start + assert result_end == expected_end + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_parse_string_to_bounds_month_feb(date_type, feb_days): + parsed = date_type(2, 2, 10, 6, 2, 8, 1) + expected_start = date_type(2, 2, 1) + expected_end = date_type(2, 2, feb_days, 23, 59, 59, 999999) + result_start, result_end = _parsed_string_to_bounds( + date_type, 'month', parsed) + assert result_start == expected_start + assert result_end == expected_end + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_parse_string_to_bounds_month_dec(date_type, dec_days): + parsed = date_type(2, 12, 1) + expected_start = date_type(2, 12, 1) + expected_end = date_type(2, 12, dec_days, 23, 59, 59, 999999) + result_start, result_end = _parsed_string_to_bounds( + date_type, 'month', parsed) + assert result_start == expected_start + assert result_end == expected_end + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize(('reso', 'ex_start_args', 'ex_end_args'), [ + ('day', (2, 2, 10), (2, 2, 10, 23, 59, 59, 999999)), + ('hour', (2, 2, 10, 6), (2, 2, 10, 6, 59, 59, 999999)), + ('minute', (2, 2, 10, 6, 2), (2, 2, 10, 6, 2, 59, 999999)), + ('second', (2, 2, 10, 6, 2, 8), (2, 2, 10, 6, 2, 8, 999999))]) +def test_parsed_string_to_bounds_sub_monthly(date_type, reso, + ex_start_args, ex_end_args): + parsed = date_type(2, 2, 10, 6, 2, 8, 123456) + expected_start = date_type(*ex_start_args) + expected_end = date_type(*ex_end_args) + + result_start, result_end = _parsed_string_to_bounds( + date_type, reso, parsed) + assert result_start == expected_start + assert result_end == expected_end + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_parsed_string_to_bounds_raises(date_type): + with pytest.raises(KeyError): + _parsed_string_to_bounds(date_type, 'a', date_type(1, 1, 1)) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_get_loc(date_type, index): + result = index.get_loc('0001') + expected = [0, 1] + assert_array_equal(result, expected) + + result = index.get_loc(date_type(1, 2, 1)) + expected = 1 + assert result == expected + + result = index.get_loc('0001-02-01') + expected = 1 + assert result == expected + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('kind', ['loc', 'getitem']) +def test_get_slice_bound(date_type, index, kind): + result = index.get_slice_bound('0001', 'left', kind) + expected = 0 + assert result == expected + + result = index.get_slice_bound('0001', 'right', kind) + expected = 2 + assert result == expected + + result = index.get_slice_bound( + date_type(1, 3, 1), 'left', kind) + expected = 2 + assert result == expected + + result = index.get_slice_bound( + date_type(1, 3, 1), 'right', kind) + expected = 2 + assert result == expected + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('kind', ['loc', 'getitem']) +def test_get_slice_bound_decreasing_index( + date_type, monotonic_decreasing_index, kind): + result = monotonic_decreasing_index.get_slice_bound('0001', 'left', kind) + expected = 2 + assert result == expected + + result = monotonic_decreasing_index.get_slice_bound('0001', 'right', kind) + expected = 4 + assert result == expected + + result = monotonic_decreasing_index.get_slice_bound( + date_type(1, 3, 1), 'left', kind) + expected = 2 + assert result == expected + + result = monotonic_decreasing_index.get_slice_bound( + date_type(1, 3, 1), 'right', kind) + expected = 2 + assert result == expected + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_date_type_property(date_type, index): + assert index.date_type is date_type + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_contains(date_type, index): + assert '0001-01-01' in index + assert '0001' in index + assert '0003' not in index + assert date_type(1, 1, 1) in index + assert date_type(3, 1, 1) not in index + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_groupby(da): + result = da.groupby('time.month').sum('time') + expected = xr.DataArray([4, 6], coords=[[1, 2]], dims=['month']) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_resample_error(da): + with pytest.raises(TypeError): + da.resample(time='Y') + + +SEL_STRING_OR_LIST_TESTS = { + 'string': '0001', + 'string-slice': slice('0001-01-01', '0001-12-30'), + 'bool-list': [True, True, False, False] +} + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_arg', list(SEL_STRING_OR_LIST_TESTS.values()), + ids=list(SEL_STRING_OR_LIST_TESTS.keys())) +def test_sel_string_or_list(da, index, sel_arg): + expected = xr.DataArray([1, 2], coords=[index[:2]], dims=['time']) + result = da.sel(time=sel_arg) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_sel_date_slice_or_list(da, index, date_type): + expected = xr.DataArray([1, 2], coords=[index[:2]], dims=['time']) + result = da.sel(time=slice(date_type(1, 1, 1), date_type(1, 12, 30))) + assert_identical(result, expected) + + result = da.sel(time=[date_type(1, 1, 1), date_type(1, 2, 1)]) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_sel_date_scalar(da, date_type, index): + expected = xr.DataArray(1).assign_coords(time=index[0]) + result = da.sel(time=date_type(1, 1, 1)) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'nearest'}, + {'method': 'nearest', 'tolerance': timedelta(days=70)} +]) +def test_sel_date_scalar_nearest(da, date_type, index, sel_kwargs): + expected = xr.DataArray(2).assign_coords(time=index[1]) + result = da.sel(time=date_type(1, 4, 1), **sel_kwargs) + assert_identical(result, expected) + + expected = xr.DataArray(3).assign_coords(time=index[2]) + result = da.sel(time=date_type(1, 11, 1), **sel_kwargs) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'pad'}, + {'method': 'pad', 'tolerance': timedelta(days=365)} +]) +def test_sel_date_scalar_pad(da, date_type, index, sel_kwargs): + expected = xr.DataArray(2).assign_coords(time=index[1]) + result = da.sel(time=date_type(1, 4, 1), **sel_kwargs) + assert_identical(result, expected) + + expected = xr.DataArray(2).assign_coords(time=index[1]) + result = da.sel(time=date_type(1, 11, 1), **sel_kwargs) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'backfill'}, + {'method': 'backfill', 'tolerance': timedelta(days=365)} +]) +def test_sel_date_scalar_backfill(da, date_type, index, sel_kwargs): + expected = xr.DataArray(3).assign_coords(time=index[2]) + result = da.sel(time=date_type(1, 4, 1), **sel_kwargs) + assert_identical(result, expected) + + expected = xr.DataArray(3).assign_coords(time=index[2]) + result = da.sel(time=date_type(1, 11, 1), **sel_kwargs) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'pad', 'tolerance': timedelta(days=20)}, + {'method': 'backfill', 'tolerance': timedelta(days=20)}, + {'method': 'nearest', 'tolerance': timedelta(days=20)}, +]) +def test_sel_date_scalar_tolerance_raises(da, date_type, sel_kwargs): + with pytest.raises(KeyError): + da.sel(time=date_type(1, 5, 1), **sel_kwargs) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'nearest'}, + {'method': 'nearest', 'tolerance': timedelta(days=70)} +]) +def test_sel_date_list_nearest(da, date_type, index, sel_kwargs): + expected = xr.DataArray( + [2, 2], coords=[[index[1], index[1]]], dims=['time']) + result = da.sel( + time=[date_type(1, 3, 1), date_type(1, 4, 1)], **sel_kwargs) + assert_identical(result, expected) + + expected = xr.DataArray( + [2, 3], coords=[[index[1], index[2]]], dims=['time']) + result = da.sel( + time=[date_type(1, 3, 1), date_type(1, 12, 1)], **sel_kwargs) + assert_identical(result, expected) + + expected = xr.DataArray( + [3, 3], coords=[[index[2], index[2]]], dims=['time']) + result = da.sel( + time=[date_type(1, 11, 1), date_type(1, 12, 1)], **sel_kwargs) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'pad'}, + {'method': 'pad', 'tolerance': timedelta(days=365)} +]) +def test_sel_date_list_pad(da, date_type, index, sel_kwargs): + expected = xr.DataArray( + [2, 2], coords=[[index[1], index[1]]], dims=['time']) + result = da.sel( + time=[date_type(1, 3, 1), date_type(1, 4, 1)], **sel_kwargs) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'backfill'}, + {'method': 'backfill', 'tolerance': timedelta(days=365)} +]) +def test_sel_date_list_backfill(da, date_type, index, sel_kwargs): + expected = xr.DataArray( + [3, 3], coords=[[index[2], index[2]]], dims=['time']) + result = da.sel( + time=[date_type(1, 3, 1), date_type(1, 4, 1)], **sel_kwargs) + assert_identical(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('sel_kwargs', [ + {'method': 'pad', 'tolerance': timedelta(days=20)}, + {'method': 'backfill', 'tolerance': timedelta(days=20)}, + {'method': 'nearest', 'tolerance': timedelta(days=20)}, +]) +def test_sel_date_list_tolerance_raises(da, date_type, sel_kwargs): + with pytest.raises(KeyError): + da.sel(time=[date_type(1, 2, 1), date_type(1, 5, 1)], **sel_kwargs) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_isel(da, index): + expected = xr.DataArray(1).assign_coords(time=index[0]) + result = da.isel(time=0) + assert_identical(result, expected) + + expected = xr.DataArray([1, 2], coords=[index[:2]], dims=['time']) + result = da.isel(time=[0, 1]) + assert_identical(result, expected) + + +@pytest.fixture +def scalar_args(date_type): + return [date_type(1, 1, 1)] + + +@pytest.fixture +def range_args(date_type): + return ['0001', slice('0001-01-01', '0001-12-30'), + slice(None, '0001-12-30'), + slice(date_type(1, 1, 1), date_type(1, 12, 30)), + slice(None, date_type(1, 12, 30))] + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_indexing_in_series_getitem(series, index, scalar_args, range_args): + for arg in scalar_args: + assert series[arg] == 1 + + expected = pd.Series([1, 2], index=index[:2]) + for arg in range_args: + assert series[arg].equals(expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_indexing_in_series_loc(series, index, scalar_args, range_args): + for arg in scalar_args: + assert series.loc[arg] == 1 + + expected = pd.Series([1, 2], index=index[:2]) + for arg in range_args: + assert series.loc[arg].equals(expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_indexing_in_series_iloc(series, index): + expected = 1 + assert series.iloc[0] == expected + + expected = pd.Series([1, 2], index=index[:2]) + assert series.iloc[:2].equals(expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_indexing_in_dataframe_loc(df, index, scalar_args, range_args): + expected = pd.Series([1], name=index[0]) + for arg in scalar_args: + result = df.loc[arg] + assert result.equals(expected) + + expected = pd.DataFrame([1, 2], index=index[:2]) + for arg in range_args: + result = df.loc[arg] + assert result.equals(expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_indexing_in_dataframe_iloc(df, index): + expected = pd.Series([1], name=index[0]) + result = df.iloc[0] + assert result.equals(expected) + assert result.equals(expected) + + expected = pd.DataFrame([1, 2], index=index[:2]) + result = df.iloc[:2] + assert result.equals(expected) + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize('enable_cftimeindex', [False, True]) +def test_concat_cftimeindex(date_type, enable_cftimeindex): + with xr.set_options(enable_cftimeindex=enable_cftimeindex): + da1 = xr.DataArray( + [1., 2.], coords=[[date_type(1, 1, 1), date_type(1, 2, 1)]], + dims=['time']) + da2 = xr.DataArray( + [3., 4.], coords=[[date_type(1, 3, 1), date_type(1, 4, 1)]], + dims=['time']) + da = xr.concat([da1, da2], dim='time') + + if enable_cftimeindex and has_cftime: + assert isinstance(da.indexes['time'], CFTimeIndex) + else: + assert isinstance(da.indexes['time'], pd.Index) + assert not isinstance(da.indexes['time'], CFTimeIndex) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 7e69d4b3ff2..7c1e869f772 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -1,15 +1,55 @@ from __future__ import absolute_import, division, print_function +from itertools import product import warnings import numpy as np import pandas as pd import pytest -from xarray import Variable, coding +from xarray import Variable, coding, set_options, DataArray, decode_cf from xarray.coding.times import _import_cftime - -from . import TestCase, assert_array_equal, requires_cftime_or_netCDF4 +from xarray.coding.variables import SerializationWarning +from xarray.core.common import contains_cftime_datetimes + +from . import (assert_array_equal, has_cftime_or_netCDF4, + requires_cftime_or_netCDF4, has_cftime, has_dask) + + +_NON_STANDARD_CALENDARS = {'noleap', '365_day', '360_day', + 'julian', 'all_leap', '366_day'} +_ALL_CALENDARS = _NON_STANDARD_CALENDARS.union( + coding.times._STANDARD_CALENDARS) +_CF_DATETIME_NUM_DATES_UNITS = [ + (np.arange(10), 'days since 2000-01-01'), + (np.arange(10).astype('float64'), 'days since 2000-01-01'), + (np.arange(10).astype('float32'), 'days since 2000-01-01'), + (np.arange(10).reshape(2, 5), 'days since 2000-01-01'), + (12300 + np.arange(5), 'hours since 1680-01-01 00:00:00'), + # here we add a couple minor formatting errors to test + # the robustness of the parsing algorithm. + (12300 + np.arange(5), 'hour since 1680-01-01 00:00:00'), + (12300 + np.arange(5), u'Hour since 1680-01-01 00:00:00'), + (12300 + np.arange(5), ' Hour since 1680-01-01 00:00:00 '), + (10, 'days since 2000-01-01'), + ([10], 'daYs since 2000-01-01'), + ([[10]], 'days since 2000-01-01'), + ([10, 10], 'days since 2000-01-01'), + (np.array(10), 'days since 2000-01-01'), + (0, 'days since 1000-01-01'), + ([0], 'days since 1000-01-01'), + ([[0]], 'days since 1000-01-01'), + (np.arange(2), 'days since 1000-01-01'), + (np.arange(0, 100000, 20000), 'days since 1900-01-01'), + (17093352.0, 'hours since 1-1-1 00:00:0.0'), + ([0.5, 1.5], 'hours since 1900-01-01T00:00:00'), + (0, 'milliseconds since 2000-01-01T00:00:00'), + (0, 'microseconds since 2000-01-01T00:00:00'), + (np.int32(788961600), 'seconds since 1981-01-01') # GH2002 +] +_CF_DATETIME_TESTS = [num_dates_units + (calendar,) for num_dates_units, + calendar in product(_CF_DATETIME_NUM_DATES_UNITS, + coding.times._STANDARD_CALENDARS)] @np.vectorize @@ -20,309 +60,698 @@ def _ensure_naive_tz(dt): return dt -class TestDatetime(TestCase): - @requires_cftime_or_netCDF4 - def test_cf_datetime(self): - cftime = _import_cftime() - for num_dates, units in [ - (np.arange(10), 'days since 2000-01-01'), - (np.arange(10).astype('float64'), 'days since 2000-01-01'), - (np.arange(10).astype('float32'), 'days since 2000-01-01'), - (np.arange(10).reshape(2, 5), 'days since 2000-01-01'), - (12300 + np.arange(5), 'hours since 1680-01-01 00:00:00'), - # here we add a couple minor formatting errors to test - # the robustness of the parsing algorithm. - (12300 + np.arange(5), 'hour since 1680-01-01 00:00:00'), - (12300 + np.arange(5), u'Hour since 1680-01-01 00:00:00'), - (12300 + np.arange(5), ' Hour since 1680-01-01 00:00:00 '), - (10, 'days since 2000-01-01'), - ([10], 'daYs since 2000-01-01'), - ([[10]], 'days since 2000-01-01'), - ([10, 10], 'days since 2000-01-01'), - (np.array(10), 'days since 2000-01-01'), - (0, 'days since 1000-01-01'), - ([0], 'days since 1000-01-01'), - ([[0]], 'days since 1000-01-01'), - (np.arange(2), 'days since 1000-01-01'), - (np.arange(0, 100000, 20000), 'days since 1900-01-01'), - (17093352.0, 'hours since 1-1-1 00:00:0.0'), - ([0.5, 1.5], 'hours since 1900-01-01T00:00:00'), - (0, 'milliseconds since 2000-01-01T00:00:00'), - (0, 'microseconds since 2000-01-01T00:00:00'), - (np.int32(788961600), 'seconds since 1981-01-01'), # GH2002 - ]: - for calendar in ['standard', 'gregorian', 'proleptic_gregorian']: - expected = _ensure_naive_tz( - cftime.num2date(num_dates, units, calendar)) - print(num_dates, units, calendar) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', - 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime(num_dates, units, - calendar) - if (isinstance(actual, np.ndarray) and - np.issubdtype(actual.dtype, np.datetime64)): - # self.assertEqual(actual.dtype.kind, 'M') - # For some reason, numpy 1.8 does not compare ns precision - # datetime64 arrays as equal to arrays of datetime objects, - # but it works for us precision. Thus, convert to us - # precision for the actual array equal comparison... - actual_cmp = actual.astype('M8[us]') - else: - actual_cmp = actual - assert_array_equal(expected, actual_cmp) - encoded, _, _ = coding.times.encode_cf_datetime(actual, units, - calendar) - if '1-1-1' not in units: - # pandas parses this date very strangely, so the original - # units/encoding cannot be preserved in this case: - # (Pdb) pd.to_datetime('1-1-1 00:00:0.0') - # Timestamp('2001-01-01 00:00:00') - assert_array_equal(num_dates, np.around(encoded, 1)) - if (hasattr(num_dates, 'ndim') and num_dates.ndim == 1 and - '1000' not in units): - # verify that wrapping with a pandas.Index works - # note that it *does not* currently work to even put - # non-datetime64 compatible dates into a pandas.Index - encoded, _, _ = coding.times.encode_cf_datetime( - pd.Index(actual), units, calendar) - assert_array_equal(num_dates, np.around(encoded, 1)) - - @requires_cftime_or_netCDF4 - def test_decode_cf_datetime_overflow(self): - # checks for - # https://github.com/pydata/pandas/issues/14068 - # https://github.com/pydata/xarray/issues/975 - - from datetime import datetime - units = 'days since 2000-01-01 00:00:00' - - # date after 2262 and before 1678 - days = (-117608, 95795) - expected = (datetime(1677, 12, 31), datetime(2262, 4, 12)) - - for i, day in enumerate(days): - result = coding.times.decode_cf_datetime(day, units) - assert result == expected[i] - - def test_decode_cf_datetime_non_standard_units(self): - expected = pd.date_range(periods=100, start='1970-01-01', freq='h') - # netCDFs from madis.noaa.gov use this format for their time units - # they cannot be parsed by netcdftime, but pd.Timestamp works - units = 'hours since 1-1-1970' - actual = coding.times.decode_cf_datetime(np.arange(100), units) +def _all_cftime_date_types(): + try: + import cftime + except ImportError: + import netcdftime as cftime + return {'noleap': cftime.DatetimeNoLeap, + '365_day': cftime.DatetimeNoLeap, + '360_day': cftime.Datetime360Day, + 'julian': cftime.DatetimeJulian, + 'all_leap': cftime.DatetimeAllLeap, + '366_day': cftime.DatetimeAllLeap, + 'gregorian': cftime.DatetimeGregorian, + 'proleptic_gregorian': cftime.DatetimeProlepticGregorian} + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize(['num_dates', 'units', 'calendar'], + _CF_DATETIME_TESTS) +def test_cf_datetime(num_dates, units, calendar): + cftime = _import_cftime() + expected = _ensure_naive_tz( + cftime.num2date(num_dates, units, calendar)) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime(num_dates, units, + calendar) + if (isinstance(actual, np.ndarray) and + np.issubdtype(actual.dtype, np.datetime64)): + # self.assertEqual(actual.dtype.kind, 'M') + # For some reason, numpy 1.8 does not compare ns precision + # datetime64 arrays as equal to arrays of datetime objects, + # but it works for us precision. Thus, convert to us + # precision for the actual array equal comparison... + actual_cmp = actual.astype('M8[us]') + else: + actual_cmp = actual + assert_array_equal(expected, actual_cmp) + encoded, _, _ = coding.times.encode_cf_datetime(actual, units, + calendar) + if '1-1-1' not in units: + # pandas parses this date very strangely, so the original + # units/encoding cannot be preserved in this case: + # (Pdb) pd.to_datetime('1-1-1 00:00:0.0') + # Timestamp('2001-01-01 00:00:00') + assert_array_equal(num_dates, np.around(encoded, 1)) + if (hasattr(num_dates, 'ndim') and num_dates.ndim == 1 and + '1000' not in units): + # verify that wrapping with a pandas.Index works + # note that it *does not* currently work to even put + # non-datetime64 compatible dates into a pandas.Index + encoded, _, _ = coding.times.encode_cf_datetime( + pd.Index(actual), units, calendar) + assert_array_equal(num_dates, np.around(encoded, 1)) + + +@requires_cftime_or_netCDF4 +def test_decode_cf_datetime_overflow(): + # checks for + # https://github.com/pydata/pandas/issues/14068 + # https://github.com/pydata/xarray/issues/975 + + from datetime import datetime + units = 'days since 2000-01-01 00:00:00' + + # date after 2262 and before 1678 + days = (-117608, 95795) + expected = (datetime(1677, 12, 31), datetime(2262, 4, 12)) + + for i, day in enumerate(days): + result = coding.times.decode_cf_datetime(day, units) + assert result == expected[i] + + +def test_decode_cf_datetime_non_standard_units(): + expected = pd.date_range(periods=100, start='1970-01-01', freq='h') + # netCDFs from madis.noaa.gov use this format for their time units + # they cannot be parsed by cftime, but pd.Timestamp works + units = 'hours since 1-1-1970' + actual = coding.times.decode_cf_datetime(np.arange(100), units) + assert_array_equal(actual, expected) + + +@requires_cftime_or_netCDF4 +def test_decode_cf_datetime_non_iso_strings(): + # datetime strings that are _almost_ ISO compliant but not quite, + # but which netCDF4.num2date can still parse correctly + expected = pd.date_range(periods=100, start='2000-01-01', freq='h') + cases = [(np.arange(100), 'hours since 2000-01-01 0'), + (np.arange(100), 'hours since 2000-1-1 0'), + (np.arange(100), 'hours since 2000-01-01 0:00')] + for num_dates, units in cases: + actual = coding.times.decode_cf_datetime(num_dates, units) assert_array_equal(actual, expected) - @requires_cftime_or_netCDF4 - def test_decode_cf_datetime_non_iso_strings(self): - # datetime strings that are _almost_ ISO compliant but not quite, - # but which netCDF4.num2date can still parse correctly - expected = pd.date_range(periods=100, start='2000-01-01', freq='h') - cases = [(np.arange(100), 'hours since 2000-01-01 0'), - (np.arange(100), 'hours since 2000-1-1 0'), - (np.arange(100), 'hours since 2000-01-01 0:00')] - for num_dates, units in cases: - actual = coding.times.decode_cf_datetime(num_dates, units) - assert_array_equal(actual, expected) - - @requires_cftime_or_netCDF4 - def test_decode_non_standard_calendar(self): - cftime = _import_cftime() - - for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', - '366_day']: - units = 'days since 0001-01-01' - times = pd.date_range('2001-04-01-00', end='2001-04-30-23', - freq='H') - noleap_time = cftime.date2num(times.to_pydatetime(), units, - calendar=calendar) - expected = times.values - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime(noleap_time, units, - calendar=calendar) - assert actual.dtype == np.dtype('M8[ns]') - abs_diff = abs(actual - expected) - # once we no longer support versions of netCDF4 older than 1.1.5, - # we could do this check with near microsecond accuracy: - # https://github.com/Unidata/netcdf4-python/issues/355 - assert (abs_diff <= np.timedelta64(1, 's')).all() - - @requires_cftime_or_netCDF4 - def test_decode_non_standard_calendar_single_element(self): - units = 'days since 0001-01-01' - for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap', - '366_day']: - for num_time in [735368, [735368], [[735368]]]: - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', - 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime(num_time, units, - calendar=calendar) - assert actual.dtype == np.dtype('M8[ns]') - - @requires_cftime_or_netCDF4 - def test_decode_non_standard_calendar_single_element_fallback(self): - cftime = _import_cftime() - - units = 'days since 0001-01-01' - try: - dt = cftime.netcdftime.datetime(2001, 2, 29) - except AttributeError: - # Must be using standalone netcdftime library - dt = cftime.datetime(2001, 2, 29) - for calendar in ['360_day', 'all_leap', '366_day']: - num_time = cftime.date2num(dt, units, calendar) - with pytest.warns(Warning, match='Unable to decode time axis'): - actual = coding.times.decode_cf_datetime(num_time, units, - calendar=calendar) - expected = np.asarray(cftime.num2date(num_time, units, calendar)) - assert actual.dtype == np.dtype('O') - assert expected == actual - - @requires_cftime_or_netCDF4 - def test_decode_non_standard_calendar_multidim_time(self): - cftime = _import_cftime() - - calendar = 'noleap' - units = 'days since 0001-01-01' - times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') - times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = cftime.date2num(times1.to_pydatetime(), units, - calendar=calendar) - noleap_time2 = cftime.date2num(times2.to_pydatetime(), units, - calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 - expected1 = times1.values - expected2 = times2.values +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(coding.times._STANDARD_CALENDARS, [False, True])) +def test_decode_standard_calendar_inside_timestamp_range( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + units = 'days since 0001-01-01' + times = pd.date_range('2001-04-01-00', end='2001-04-30-23', + freq='H') + noleap_time = cftime.date2num(times.to_pydatetime(), units, + calendar=calendar) + expected = times.values + expected_dtype = np.dtype('M8[ns]') + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime( + noleap_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + assert actual.dtype == expected_dtype + abs_diff = abs(actual - expected) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(_NON_STANDARD_CALENDARS, [False, True])) +def test_decode_non_standard_calendar_inside_timestamp_range( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + units = 'days since 0001-01-01' + times = pd.date_range('2001-04-01-00', end='2001-04-30-23', + freq='H') + noleap_time = cftime.date2num(times.to_pydatetime(), units, + calendar=calendar) + if enable_cftimeindex: + expected = cftime.num2date(noleap_time, units, calendar=calendar) + expected_dtype = np.dtype('O') + else: + expected = times.values + expected_dtype = np.dtype('M8[ns]') + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime( + noleap_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + assert actual.dtype == expected_dtype + abs_diff = abs(actual - expected) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(_ALL_CALENDARS, [False, True])) +def test_decode_dates_outside_timestamp_range( + calendar, enable_cftimeindex): + from datetime import datetime + + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + + units = 'days since 0001-01-01' + times = [datetime(1, 4, 1, h) for h in range(1, 5)] + noleap_time = cftime.date2num(times, units, calendar=calendar) + if enable_cftimeindex: + expected = cftime.num2date(noleap_time, units, calendar=calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(noleap_time, units, calendar=calendar) + expected_date_type = type(expected[0]) + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime( + noleap_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + assert all(isinstance(value, expected_date_type) for value in actual) + abs_diff = abs(actual - expected) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(coding.times._STANDARD_CALENDARS, [False, True])) +def test_decode_standard_calendar_single_element_inside_timestamp_range( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + units = 'days since 0001-01-01' + for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime(mdim_time, units, - calendar=calendar) + warnings.filterwarnings('ignore', + 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) assert actual.dtype == np.dtype('M8[ns]') - assert_array_equal(actual[:, 0], expected1) - assert_array_equal(actual[:, 1], expected2) - - @requires_cftime_or_netCDF4 - def test_decode_non_standard_calendar_fallback(self): - cftime = _import_cftime() - # ensure leap year doesn't matter - for year in [2010, 2011, 2012, 2013, 2014]: - for calendar in ['360_day', '366_day', 'all_leap']: - calendar = '360_day' - units = 'days since {0}-01-01'.format(year) - num_times = np.arange(100) - expected = cftime.num2date(num_times, units, calendar) - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - actual = coding.times.decode_cf_datetime(num_times, units, - calendar=calendar) - assert len(w) == 1 - assert 'Unable to decode time axis' in \ - str(w[0].message) - - assert actual.dtype == np.dtype('O') - assert_array_equal(actual, expected) - - @requires_cftime_or_netCDF4 - def test_cf_datetime_nan(self): - for num_dates, units, expected_list in [ - ([np.nan], 'days since 2000-01-01', ['NaT']), - ([np.nan, 0], 'days since 2000-01-01', - ['NaT', '2000-01-01T00:00:00Z']), - ([np.nan, 0, 1], 'days since 2000-01-01', - ['NaT', '2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z']), - ]: + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(_NON_STANDARD_CALENDARS, [False, True])) +def test_decode_non_standard_calendar_single_element_inside_timestamp_range( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + units = 'days since 0001-01-01' + for num_time in [735368, [735368], [[735368]]]: + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + if enable_cftimeindex: + assert actual.dtype == np.dtype('O') + else: + assert actual.dtype == np.dtype('M8[ns]') + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(_NON_STANDARD_CALENDARS, [False, True])) +def test_decode_single_element_outside_timestamp_range( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + units = 'days since 0001-01-01' + for days in [1, 1470376]: + for num_time in [days, [days], [[days]]]: with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'All-NaN') - actual = coding.times.decode_cf_datetime(num_dates, units) - expected = np.array(expected_list, dtype='datetime64[ns]') - assert_array_equal(expected, actual) - - @requires_cftime_or_netCDF4 - def test_decoded_cf_datetime_array_2d(self): - # regression test for GH1229 - variable = Variable(('x', 'y'), np.array([[0, 1], [2, 3]]), - {'units': 'days since 2000-01-01'}) - result = coding.times.CFDatetimeCoder().decode(variable) - assert result.dtype == 'datetime64[ns]' - expected = pd.date_range('2000-01-01', periods=4).values.reshape(2, 2) - assert_array_equal(np.asarray(result), expected) - - def test_infer_datetime_units(self): - for dates, expected in [(pd.date_range('1900-01-01', periods=5), - 'days since 1900-01-01 00:00:00'), - (pd.date_range('1900-01-01 12:00:00', freq='H', - periods=2), - 'hours since 1900-01-01 12:00:00'), - (['1900-01-01', '1900-01-02', - '1900-01-02 00:00:01'], - 'seconds since 1900-01-01 00:00:00'), - (pd.to_datetime( - ['1900-01-01', '1900-01-02', 'NaT']), - 'days since 1900-01-01 00:00:00'), - (pd.to_datetime(['1900-01-01', - '1900-01-02T00:00:00.005']), - 'seconds since 1900-01-01 00:00:00'), - (pd.to_datetime(['NaT', '1900-01-01']), - 'days since 1900-01-01 00:00:00'), - (pd.to_datetime(['NaT']), - 'days since 1970-01-01 00:00:00'), - ]: - assert expected == coding.times.infer_datetime_units(dates) + warnings.filterwarnings('ignore', + 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + expected = cftime.num2date(days, units, calendar) + assert isinstance(actual.item(), type(expected)) + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(coding.times._STANDARD_CALENDARS, [False, True])) +def test_decode_standard_calendar_multidim_time_inside_timestamp_range( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + + units = 'days since 0001-01-01' + times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') + times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') + noleap_time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + noleap_time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(noleap_time1), 2), ) + mdim_time[:, 0] = noleap_time1 + mdim_time[:, 1] = noleap_time2 + + expected1 = times1.values + expected2 = times2.values + + actual = coding.times.decode_cf_datetime( + mdim_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + assert actual.dtype == np.dtype('M8[ns]') + + abs_diff1 = abs(actual[:, 0] - expected1) + abs_diff2 = abs(actual[:, 1] - expected2) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff1 <= np.timedelta64(1, 's')).all() + assert (abs_diff2 <= np.timedelta64(1, 's')).all() + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(_NON_STANDARD_CALENDARS, [False, True])) +def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + + units = 'days since 0001-01-01' + times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') + times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') + noleap_time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + noleap_time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(noleap_time1), 2), ) + mdim_time[:, 0] = noleap_time1 + mdim_time[:, 1] = noleap_time2 + + if enable_cftimeindex: + expected1 = cftime.num2date(noleap_time1, units, calendar) + expected2 = cftime.num2date(noleap_time2, units, calendar) + expected_dtype = np.dtype('O') + else: + expected1 = times1.values + expected2 = times2.values + expected_dtype = np.dtype('M8[ns]') + + actual = coding.times.decode_cf_datetime( + mdim_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + + assert actual.dtype == expected_dtype + abs_diff1 = abs(actual[:, 0] - expected1) + abs_diff2 = abs(actual[:, 1] - expected2) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff1 <= np.timedelta64(1, 's')).all() + assert (abs_diff2 <= np.timedelta64(1, 's')).all() + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(_ALL_CALENDARS, [False, True])) +def test_decode_multidim_time_outside_timestamp_range( + calendar, enable_cftimeindex): + from datetime import datetime + + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + + units = 'days since 0001-01-01' + times1 = [datetime(1, 4, day) for day in range(1, 6)] + times2 = [datetime(1, 5, day) for day in range(1, 6)] + noleap_time1 = cftime.date2num(times1, units, calendar=calendar) + noleap_time2 = cftime.date2num(times2, units, calendar=calendar) + mdim_time = np.empty((len(noleap_time1), 2), ) + mdim_time[:, 0] = noleap_time1 + mdim_time[:, 1] = noleap_time2 + + if enable_cftimeindex: + expected1 = cftime.num2date(noleap_time1, units, calendar, + only_use_cftime_datetimes=True) + expected2 = cftime.num2date(noleap_time2, units, calendar, + only_use_cftime_datetimes=True) + else: + expected1 = cftime.num2date(noleap_time1, units, calendar) + expected2 = cftime.num2date(noleap_time2, units, calendar) + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + actual = coding.times.decode_cf_datetime( + mdim_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + + assert actual.dtype == np.dtype('O') + + abs_diff1 = abs(actual[:, 0] - expected1) + abs_diff2 = abs(actual[:, 1] - expected2) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff1 <= np.timedelta64(1, 's')).all() + assert (abs_diff2 <= np.timedelta64(1, 's')).all() + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(['360_day', 'all_leap', '366_day'], [False, True])) +def test_decode_non_standard_calendar_single_element_fallback( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + + units = 'days since 0001-01-01' + try: + dt = cftime.netcdftime.datetime(2001, 2, 29) + except AttributeError: + # Must be using standalone netcdftime library + dt = cftime.datetime(2001, 2, 29) + + num_time = cftime.date2num(dt, units, calendar) + if enable_cftimeindex: + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + else: + with pytest.warns(SerializationWarning, + match='Unable to decode time axis'): + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + + expected = np.asarray(cftime.num2date(num_time, units, calendar)) + assert actual.dtype == np.dtype('O') + assert expected == actual + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(['360_day'], [False, True])) +def test_decode_non_standard_calendar_fallback( + calendar, enable_cftimeindex): + if enable_cftimeindex: + pytest.importorskip('cftime') + + cftime = _import_cftime() + # ensure leap year doesn't matter + for year in [2010, 2011, 2012, 2013, 2014]: + units = 'days since {0}-01-01'.format(year) + num_times = np.arange(100) + expected = cftime.num2date(num_times, units, calendar) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + actual = coding.times.decode_cf_datetime( + num_times, units, calendar=calendar, + enable_cftimeindex=enable_cftimeindex) + if enable_cftimeindex: + assert len(w) == 0 + else: + assert len(w) == 1 + assert 'Unable to decode time axis' in str(w[0].message) + + assert actual.dtype == np.dtype('O') + assert_array_equal(actual, expected) - def test_cf_timedelta(self): - examples = [ - ('1D', 'days', np.int64(1)), - (['1D', '2D', '3D'], 'days', np.array([1, 2, 3], 'int64')), - ('1h', 'hours', np.int64(1)), - ('1ms', 'milliseconds', np.int64(1)), - ('1us', 'microseconds', np.int64(1)), - (['NaT', '0s', '1s'], None, [np.nan, 0, 1]), - (['30m', '60m'], 'hours', [0.5, 1.0]), - (np.timedelta64('NaT', 'ns'), 'days', np.nan), - (['NaT', 'NaT'], 'days', [np.nan, np.nan]), - ] - - for timedeltas, units, numbers in examples: - timedeltas = pd.to_timedelta(timedeltas, box=False) - numbers = np.array(numbers) - - expected = numbers - actual, _ = coding.times.encode_cf_timedelta(timedeltas, units) - assert_array_equal(expected, actual) - assert expected.dtype == actual.dtype - - if units is not None: - expected = timedeltas - actual = coding.times.decode_cf_timedelta(numbers, units) - assert_array_equal(expected, actual) - assert expected.dtype == actual.dtype - - expected = np.timedelta64('NaT', 'ns') - actual = coding.times.decode_cf_timedelta(np.array(np.nan), 'days') - assert_array_equal(expected, actual) - def test_cf_timedelta_2d(self): - timedeltas = ['1D', '2D', '3D'] - units = 'days' - numbers = np.atleast_2d([1, 2, 3]) +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['num_dates', 'units', 'expected_list'], + [([np.nan], 'days since 2000-01-01', ['NaT']), + ([np.nan, 0], 'days since 2000-01-01', + ['NaT', '2000-01-01T00:00:00Z']), + ([np.nan, 0, 1], 'days since 2000-01-01', + ['NaT', '2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z'])]) +def test_cf_datetime_nan(num_dates, units, expected_list): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'All-NaN') + actual = coding.times.decode_cf_datetime(num_dates, units) + expected = np.array(expected_list, dtype='datetime64[ns]') + assert_array_equal(expected, actual) + + +@requires_cftime_or_netCDF4 +def test_decoded_cf_datetime_array_2d(): + # regression test for GH1229 + variable = Variable(('x', 'y'), np.array([[0, 1], [2, 3]]), + {'units': 'days since 2000-01-01'}) + result = coding.times.CFDatetimeCoder().decode(variable) + assert result.dtype == 'datetime64[ns]' + expected = pd.date_range('2000-01-01', periods=4).values.reshape(2, 2) + assert_array_equal(np.asarray(result), expected) + + +@pytest.mark.parametrize( + ['dates', 'expected'], + [(pd.date_range('1900-01-01', periods=5), + 'days since 1900-01-01 00:00:00'), + (pd.date_range('1900-01-01 12:00:00', freq='H', + periods=2), + 'hours since 1900-01-01 12:00:00'), + (pd.to_datetime( + ['1900-01-01', '1900-01-02', 'NaT']), + 'days since 1900-01-01 00:00:00'), + (pd.to_datetime(['1900-01-01', + '1900-01-02T00:00:00.005']), + 'seconds since 1900-01-01 00:00:00'), + (pd.to_datetime(['NaT', '1900-01-01']), + 'days since 1900-01-01 00:00:00'), + (pd.to_datetime(['NaT']), + 'days since 1970-01-01 00:00:00')]) +def test_infer_datetime_units(dates, expected): + assert expected == coding.times.infer_datetime_units(dates) + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +def test_infer_cftime_datetime_units(): + date_types = _all_cftime_date_types() + for date_type in date_types.values(): + for dates, expected in [ + ([date_type(1900, 1, 1), + date_type(1900, 1, 2)], + 'days since 1900-01-01 00:00:00.000000'), + ([date_type(1900, 1, 1, 12), + date_type(1900, 1, 1, 13)], + 'seconds since 1900-01-01 12:00:00.000000'), + ([date_type(1900, 1, 1), + date_type(1900, 1, 2), + date_type(1900, 1, 2, 0, 0, 1)], + 'seconds since 1900-01-01 00:00:00.000000'), + ([date_type(1900, 1, 1), + date_type(1900, 1, 2, 0, 0, 0, 5)], + 'days since 1900-01-01 00:00:00.000000')]: + assert expected == coding.times.infer_datetime_units(dates) - timedeltas = np.atleast_2d(pd.to_timedelta(timedeltas, box=False)) - expected = timedeltas +@pytest.mark.parametrize( + ['timedeltas', 'units', 'numbers'], + [('1D', 'days', np.int64(1)), + (['1D', '2D', '3D'], 'days', np.array([1, 2, 3], 'int64')), + ('1h', 'hours', np.int64(1)), + ('1ms', 'milliseconds', np.int64(1)), + ('1us', 'microseconds', np.int64(1)), + (['NaT', '0s', '1s'], None, [np.nan, 0, 1]), + (['30m', '60m'], 'hours', [0.5, 1.0]), + (np.timedelta64('NaT', 'ns'), 'days', np.nan), + (['NaT', 'NaT'], 'days', [np.nan, np.nan])]) +def test_cf_timedelta(timedeltas, units, numbers): + timedeltas = pd.to_timedelta(timedeltas, box=False) + numbers = np.array(numbers) + + expected = numbers + actual, _ = coding.times.encode_cf_timedelta(timedeltas, units) + assert_array_equal(expected, actual) + assert expected.dtype == actual.dtype + + if units is not None: + expected = timedeltas actual = coding.times.decode_cf_timedelta(numbers, units) assert_array_equal(expected, actual) assert expected.dtype == actual.dtype - def test_infer_timedelta_units(self): - for deltas, expected in [ - (pd.to_timedelta(['1 day', '2 days']), 'days'), - (pd.to_timedelta(['1h', '1 day 1 hour']), 'hours'), - (pd.to_timedelta(['1m', '2m', np.nan]), 'minutes'), - (pd.to_timedelta(['1m3s', '1m4s']), 'seconds')]: - assert expected == coding.times.infer_timedelta_units(deltas) + expected = np.timedelta64('NaT', 'ns') + actual = coding.times.decode_cf_timedelta(np.array(np.nan), 'days') + assert_array_equal(expected, actual) + + +def test_cf_timedelta_2d(): + timedeltas = ['1D', '2D', '3D'] + units = 'days' + numbers = np.atleast_2d([1, 2, 3]) + + timedeltas = np.atleast_2d(pd.to_timedelta(timedeltas, box=False)) + expected = timedeltas + + actual = coding.times.decode_cf_timedelta(numbers, units) + assert_array_equal(expected, actual) + assert expected.dtype == actual.dtype + + +@pytest.mark.parametrize( + ['deltas', 'expected'], + [(pd.to_timedelta(['1 day', '2 days']), 'days'), + (pd.to_timedelta(['1h', '1 day 1 hour']), 'hours'), + (pd.to_timedelta(['1m', '2m', np.nan]), 'minutes'), + (pd.to_timedelta(['1m3s', '1m4s']), 'seconds')]) +def test_infer_timedelta_units(deltas, expected): + assert expected == coding.times.infer_timedelta_units(deltas) + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize(['date_args', 'expected'], + [((1, 2, 3, 4, 5, 6), + '0001-02-03 04:05:06.000000'), + ((10, 2, 3, 4, 5, 6), + '0010-02-03 04:05:06.000000'), + ((100, 2, 3, 4, 5, 6), + '0100-02-03 04:05:06.000000'), + ((1000, 2, 3, 4, 5, 6), + '1000-02-03 04:05:06.000000')]) +def test_format_cftime_datetime(date_args, expected): + date_types = _all_cftime_date_types() + for date_type in date_types.values(): + result = coding.times.format_cftime_datetime(date_type(*date_args)) + assert result == expected + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize( + ['calendar', 'enable_cftimeindex'], + product(_ALL_CALENDARS, [False, True])) +def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): + days = [1., 2., 3.] + da = DataArray(days, coords=[days], dims=['time'], name='test') + ds = da.to_dataset() + + for v in ['test', 'time']: + ds[v].attrs['units'] = 'days since 2001-01-01' + ds[v].attrs['calendar'] = calendar + + if (not has_cftime and enable_cftimeindex and + calendar not in coding.times._STANDARD_CALENDARS): + with pytest.raises(ValueError): + with set_options(enable_cftimeindex=enable_cftimeindex): + ds = decode_cf(ds) + else: + with set_options(enable_cftimeindex=enable_cftimeindex): + ds = decode_cf(ds) + + if (enable_cftimeindex and + calendar not in coding.times._STANDARD_CALENDARS): + assert ds.test.dtype == np.dtype('O') + else: + assert ds.test.dtype == np.dtype('M8[ns]') + + +@pytest.fixture(params=_ALL_CALENDARS) +def calendar(request): + return request.param + + +@pytest.fixture() +def times(calendar): + cftime = _import_cftime() + + return cftime.num2date( + np.arange(4), units='hours since 2000-01-01', calendar=calendar, + only_use_cftime_datetimes=True) + + +@pytest.fixture() +def data(times): + data = np.random.rand(2, 2, 4) + lons = np.linspace(0, 11, 2) + lats = np.linspace(0, 20, 2) + return DataArray(data, coords=[lons, lats, times], + dims=['lon', 'lat', 'time'], name='data') + + +@pytest.fixture() +def times_3d(times): + lons = np.linspace(0, 11, 2) + lats = np.linspace(0, 20, 2) + times_arr = np.random.choice(times, size=(2, 2, 4)) + return DataArray(times_arr, coords=[lons, lats, times], + dims=['lon', 'lat', 'time'], + name='data') + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_contains_cftime_datetimes_1d(data): + assert contains_cftime_datetimes(data.time) + + +@pytest.mark.skipif(not has_dask, reason='dask not installed') +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_contains_cftime_datetimes_dask_1d(data): + assert contains_cftime_datetimes(data.time.chunk()) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_contains_cftime_datetimes_3d(times_3d): + assert contains_cftime_datetimes(times_3d) + + +@pytest.mark.skipif(not has_dask, reason='dask not installed') +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_contains_cftime_datetimes_dask_3d(times_3d): + assert contains_cftime_datetimes(times_3d.chunk()) + + +@pytest.mark.parametrize('non_cftime_data', [DataArray([]), DataArray([1, 2])]) +def test_contains_cftime_datetimes_non_cftimes(non_cftime_data): + assert not contains_cftime_datetimes(non_cftime_data) + + +@pytest.mark.skipif(not has_dask, reason='dask not installed') +@pytest.mark.parametrize('non_cftime_data', [DataArray([]), DataArray([1, 2])]) +def test_contains_cftime_datetimes_non_cftimes_dask(non_cftime_data): + assert not contains_cftime_datetimes(non_cftime_data.chunk()) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index e9a2babfa2e..b16cb8ddcea 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -11,14 +11,14 @@ import xarray as xr from xarray import ( - DataArray, Dataset, IndexVariable, Variable, align, broadcast) -from xarray.coding.times import CFDatetimeCoder + DataArray, Dataset, IndexVariable, Variable, align, broadcast, set_options) +from xarray.coding.times import CFDatetimeCoder, _import_cftime from xarray.core.common import full_like from xarray.core.pycompat import OrderedDict, iteritems from xarray.tests import ( ReturnItem, TestCase, assert_allclose, assert_array_equal, assert_equal, assert_identical, raises_regex, requires_bottleneck, requires_dask, - requires_scipy, source_ndarray, unittest) + requires_scipy, source_ndarray, unittest, requires_cftime) class TestDataArray(TestCase): @@ -2208,6 +2208,19 @@ def test_resample(self): with raises_regex(ValueError, 'index must be monotonic'): array[[2, 0, 1]].resample(time='1D') + @requires_cftime + def test_resample_cftimeindex(self): + cftime = _import_cftime() + times = cftime.num2date(np.arange(12), units='hours since 0001-01-01', + calendar='noleap') + with set_options(enable_cftimeindex=True): + array = DataArray(np.arange(12), [('time', times)]) + + with raises_regex(TypeError, + 'Only valid with DatetimeIndex, ' + 'TimedeltaIndex or PeriodIndex'): + array.resample(time='6H').mean() + def test_resample_first(self): times = pd.date_range('2000-01-01', freq='6H', periods=10) array = DataArray(np.arange(10), [('time', times)]) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index aadc452b8a7..70ed1156643 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -9,6 +9,7 @@ import xarray.plot as xplt from xarray import DataArray +from xarray.coding.times import _import_cftime from xarray.plot.plot import _infer_interval_breaks from xarray.plot.utils import ( _build_discrete_cmap, _color_palette, _determine_cmap_params, @@ -16,7 +17,7 @@ from . import ( TestCase, assert_array_equal, assert_equal, raises_regex, - requires_matplotlib, requires_seaborn) + requires_matplotlib, requires_seaborn, requires_cftime) # import mpl and change the backend before other mpl imports try: @@ -1504,3 +1505,24 @@ def test_plot_seaborn_no_import_warning(): with pytest.warns(None) as record: _color_palette('Blues', 4) assert len(record) == 0 + + +@requires_cftime +def test_plot_cftime_coordinate_error(): + cftime = _import_cftime() + time = cftime.num2date(np.arange(5), units='days since 0001-01-01', + calendar='noleap') + data = DataArray(np.arange(5), coords=[time], dims=['time']) + with raises_regex(TypeError, + 'requires coordinates to be numeric or dates'): + data.plot() + + +@requires_cftime +def test_plot_cftime_data_error(): + cftime = _import_cftime() + data = cftime.num2date(np.arange(5), units='days since 0001-01-01', + calendar='noleap') + data = DataArray(data, coords=[np.arange(5)], dims=['x']) + with raises_regex(NotImplementedError, 'cftime.datetime'): + data.plot() diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 3a76b6e8c92..0b3b0ee7dd6 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -4,10 +4,14 @@ import pandas as pd import pytest +from datetime import datetime +from xarray.coding.cftimeindex import CFTimeIndex from xarray.core import duck_array_ops, utils +from xarray.core.options import set_options from xarray.core.pycompat import OrderedDict - -from . import TestCase, assert_array_equal, requires_dask +from .test_coding_times import _all_cftime_date_types +from . import (TestCase, requires_dask, assert_array_equal, + has_cftime_or_netCDF4, has_cftime) class TestAlias(TestCase): @@ -20,20 +24,51 @@ def new_method(): old_method() -class TestSafeCastToIndex(TestCase): - def test(self): - dates = pd.date_range('2000-01-01', periods=10) - x = np.arange(5) - td = x * np.timedelta64(1, 'D') - for expected, array in [ - (dates, dates.values), - (pd.Index(x, dtype=object), x.astype(object)), - (pd.Index(td), td), - (pd.Index(td, dtype=object), td.astype(object)), - ]: - actual = utils.safe_cast_to_index(array) - assert_array_equal(expected, actual) - assert expected.dtype == actual.dtype +def test_safe_cast_to_index(): + dates = pd.date_range('2000-01-01', periods=10) + x = np.arange(5) + td = x * np.timedelta64(1, 'D') + for expected, array in [ + (dates, dates.values), + (pd.Index(x, dtype=object), x.astype(object)), + (pd.Index(td), td), + (pd.Index(td, dtype=object), td.astype(object)), + ]: + actual = utils.safe_cast_to_index(array) + assert_array_equal(expected, actual) + assert expected.dtype == actual.dtype + + +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize('enable_cftimeindex', [False, True]) +def test_safe_cast_to_index_cftimeindex(enable_cftimeindex): + date_types = _all_cftime_date_types() + for date_type in date_types.values(): + dates = [date_type(1, 1, day) for day in range(1, 20)] + + if enable_cftimeindex and has_cftime: + expected = CFTimeIndex(dates) + else: + expected = pd.Index(dates) + + with set_options(enable_cftimeindex=enable_cftimeindex): + actual = utils.safe_cast_to_index(np.array(dates)) + assert_array_equal(expected, actual) + assert expected.dtype == actual.dtype + assert isinstance(actual, type(expected)) + + +# Test that datetime.datetime objects are never used in a CFTimeIndex +@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') +@pytest.mark.parametrize('enable_cftimeindex', [False, True]) +def test_safe_cast_to_index_datetime_datetime(enable_cftimeindex): + dates = [datetime(1, 1, day) for day in range(1, 20)] + + expected = pd.Index(dates) + with set_options(enable_cftimeindex=enable_cftimeindex): + actual = utils.safe_cast_to_index(np.array(dates)) + assert_array_equal(expected, actual) + assert isinstance(actual, pd.Index) def test_multiindex_from_product_levels(): From 91ac573e00538e0372cf9e5f2fdc1528a4ee8cb8 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 13 May 2018 07:56:54 -0400 Subject: [PATCH 103/282] Add cftime to doc/environment.yml (#2126) --- doc/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/environment.yml b/doc/environment.yml index 880151ab2d9..a7683ff1824 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -21,3 +21,4 @@ dependencies: - zarr - iris - flake8 + - cftime From f861186cbd11bdbfb2aab8289118a59283a2d7af Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Mon, 14 May 2018 07:37:47 +0900 Subject: [PATCH 104/282] Reduce pad size in rolling (#2125) --- asv_bench/benchmarks/rolling.py | 9 +++++---- doc/whats-new.rst | 5 ++++- xarray/core/dask_array_ops.py | 2 +- xarray/core/rolling.py | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/asv_bench/benchmarks/rolling.py b/asv_bench/benchmarks/rolling.py index 3f2a38104de..5ba7406f6e0 100644 --- a/asv_bench/benchmarks/rolling.py +++ b/asv_bench/benchmarks/rolling.py @@ -35,7 +35,7 @@ def setup(self, *args, **kwargs): @parameterized(['func', 'center'], (['mean', 'count'], [True, False])) def time_rolling(self, func, center): - getattr(self.ds.rolling(x=window, center=center), func)() + getattr(self.ds.rolling(x=window, center=center), func)().load() @parameterized(['func', 'pandas'], (['mean', 'count'], [True, False])) @@ -44,19 +44,20 @@ def time_rolling_long(self, func, pandas): se = self.da_long.to_series() getattr(se.rolling(window=window), func)() else: - getattr(self.da_long.rolling(x=window), func)() + getattr(self.da_long.rolling(x=window), func)().load() @parameterized(['window_', 'min_periods'], ([20, 40], [5, None])) def time_rolling_np(self, window_, min_periods): self.ds.rolling(x=window_, center=False, - min_periods=min_periods).reduce(getattr(np, 'nanmean')) + min_periods=min_periods).reduce( + getattr(np, 'nanmean')).load() @parameterized(['center', 'stride'], ([True, False], [1, 200])) def time_rolling_construct(self, center, stride): self.ds.rolling(x=window, center=center).construct( - 'window_dim', stride=stride).mean(dim='window_dim') + 'window_dim', stride=stride).mean(dim='window_dim').load() class RollingDask(Rolling): diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fc5f8bf3266..49d39bacbf8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -34,6 +34,9 @@ v0.10.4 (unreleased) Enhancements ~~~~~~~~~~~~ +- Slight modification in `rolling` with dask.array and bottleneck. Also, fixed a bug in rolling an + integer dask array. + By `Keisuke Fujii `_. - Add an option for using a ``CFTimeIndex`` for indexing times with non-standard calendars and/or outside the Timestamp-valid range; this index enables a subset of the functionality of a standard @@ -43,7 +46,7 @@ Enhancements - Allow for serialization of ``cftime.datetime`` objects (:issue:`789`, :issue:`1084`, :issue:`2008`, :issue:`1252`) using the standalone ``cftime`` library. By `Spencer Clark - `_. + `_. - Support writing lists of strings as netCDF attributes (:issue:`2044`). By `Dan Nowacki `_. - :py:meth:`~xarray.Dataset.to_netcdf(engine='h5netcdf')` now accepts h5py diff --git a/xarray/core/dask_array_ops.py b/xarray/core/dask_array_ops.py index ee87c3564cc..55ba1c1cbc6 100644 --- a/xarray/core/dask_array_ops.py +++ b/xarray/core/dask_array_ops.py @@ -19,7 +19,7 @@ def dask_rolling_wrapper(moving_func, a, window, min_count=None, axis=-1): if axis < 0: axis = a.ndim + axis depth = {d: 0 for d in range(a.ndim)} - depth[axis] = window - 1 + depth[axis] = (window + 1) // 2 boundary = {d: fill_value for d in range(a.ndim)} # create ghosted arrays ag = da.ghost.ghost(a, depth=depth, boundary=boundary) diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index f54a4c36631..24ed280b19e 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -294,8 +294,8 @@ def wrapped_func(self, **kwargs): if isinstance(padded.data, dask_array_type): # Workaround to make the padded chunk size is larger than # self.window-1 - shift = - (self.window - 1) - offset = -shift - self.window // 2 + shift = - (self.window + 1) // 2 + offset = (self.window - 1) // 2 valid = (slice(None), ) * axis + ( slice(offset, offset + self.obj.shape[axis]), ) else: From d1b669ec7a1e9a0b9296855f71de72c975ec78e5 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Mon, 14 May 2018 17:09:03 +0100 Subject: [PATCH 105/282] Correct two github URLs. (#2130) --- doc/examples/multidimensional-coords.rst | 2 +- examples/xarray_multidimensional_coords.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/examples/multidimensional-coords.rst b/doc/examples/multidimensional-coords.rst index a54e6058921..eed818ba064 100644 --- a/doc/examples/multidimensional-coords.rst +++ b/doc/examples/multidimensional-coords.rst @@ -3,7 +3,7 @@ Working with Multidimensional Coordinates ========================================= -Author: `Ryan Abernathey `__ +Author: `Ryan Abernathey `__ Many datasets have *physical coordinates* which differ from their *logical coordinates*. Xarray provides several ways to plot and analyze diff --git a/examples/xarray_multidimensional_coords.ipynb b/examples/xarray_multidimensional_coords.ipynb index bed7e8b962f..6bd942c5ba2 100644 --- a/examples/xarray_multidimensional_coords.ipynb +++ b/examples/xarray_multidimensional_coords.ipynb @@ -6,7 +6,7 @@ "source": [ "# Working with Multidimensional Coordinates\n", "\n", - "Author: [Ryan Abernathey](http://github.org/rabernat)\n", + "Author: [Ryan Abernathey](https://github.com/rabernat)\n", "\n", "Many datasets have _physical coordinates_ which differ from their _logical coordinates_. Xarray provides several ways to plot and analyze such datasets." ] From 188141fe97a5effacf32f2508fd05b644c720e5d Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 14 May 2018 15:17:36 -0400 Subject: [PATCH 106/282] Fix datetime.timedelta casting bug in coding.times.infer_datetime_units (#2128) * Fix #2127 * Fix typo in time-series.rst * Use pd.to_timedelta to convert to np.timedelta64 objects * Install cftime through netcdf4 through pip * box=False --- ci/requirements-py27-windows.yml | 3 +-- doc/time-series.rst | 2 +- xarray/coding/times.py | 6 +++++- xarray/tests/test_coding_times.py | 3 +++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ci/requirements-py27-windows.yml b/ci/requirements-py27-windows.yml index 7562874785b..967b7c584b9 100644 --- a/ci/requirements-py27-windows.yml +++ b/ci/requirements-py27-windows.yml @@ -8,7 +8,6 @@ dependencies: - h5py - h5netcdf - matplotlib - - netcdf4 - pathlib2 - pytest - flake8 @@ -21,4 +20,4 @@ dependencies: - rasterio - zarr - pip: - - cftime + - netcdf4 diff --git a/doc/time-series.rst b/doc/time-series.rst index 5b857789629..a7ce9226d4d 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -73,7 +73,7 @@ native representation of dates to those that fall between the years 1678 and returned as arrays of ``cftime.datetime`` objects and a ``CFTimeIndex`` can be used for indexing. The ``CFTimeIndex`` enables only a subset of the indexing functionality of a ``pandas.DatetimeIndex`` and is only enabled -when using standalone version of ``cftime`` (not the version packaged with +when using the standalone version of ``cftime`` (not the version packaged with earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more information. Datetime indexing diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 61314d9cbe6..d946e2ed378 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -253,7 +253,11 @@ def infer_datetime_units(dates): else: reference_date = dates[0] if len(dates) > 0 else '1970-01-01' reference_date = format_cftime_datetime(reference_date) - unique_timedeltas = np.unique(np.diff(dates)).astype('timedelta64[ns]') + unique_timedeltas = np.unique(np.diff(dates)) + if unique_timedeltas.dtype == np.dtype('O'): + # Convert to np.timedelta64 objects using pandas to work around a + # NumPy casting bug: https://github.com/numpy/numpy/issues/11096 + unique_timedeltas = pd.to_timedelta(unique_timedeltas, box=False) units = _infer_time_units_from_diff(unique_timedeltas) return '%s since %s' % (units, reference_date) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 7c1e869f772..6329e91ac78 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -587,6 +587,9 @@ def test_infer_cftime_datetime_units(): 'seconds since 1900-01-01 00:00:00.000000'), ([date_type(1900, 1, 1), date_type(1900, 1, 2, 0, 0, 0, 5)], + 'days since 1900-01-01 00:00:00.000000'), + ([date_type(1900, 1, 1), date_type(1900, 1, 8), + date_type(1900, 1, 16)], 'days since 1900-01-01 00:00:00.000000')]: assert expected == coding.times.infer_datetime_units(dates) From 29d608af6694b37feac48cf369fa547d9fe2d00a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 14 May 2018 15:04:30 -0600 Subject: [PATCH 107/282] Add "awesome xarray" list to faq. (#2118) * Add "awesome xarray" list to faq. * Add whats new entry + bugfix earlier entry. * Fixes + add xrft, xmitgcm. * Add link to xarray github topic. * Add more links plus add some organization. * Remove "points of reference" sentence * Remove subheadings under geosciences. * whats-new bugfix. --- doc/faq.rst | 63 +++++++++++++++++++++++++++++++++++++++++++++++ doc/internals.rst | 17 ------------- doc/whats-new.rst | 10 ++++++-- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 68670d0f5a4..46f1e20f4e8 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -157,6 +157,69 @@ and CDAT have some great domain specific functionality, and we would love to have support for converting their native objects to and from xarray (see :issue:`37` and :issue:`133`) + +What other projects leverage xarray? +------------------------------------ + +Here are several existing libraries that build functionality upon xarray. + +Geosciences +~~~~~~~~~~~ + +- `aospy `_: Automated analysis and management of gridded climate data. +- `infinite-diff `_: xarray-based finite-differencing, focused on gridded climate/meterology data +- `marc_analysis `_: Analysis package for CESM/MARC experiments and output. +- `MPAS-Analysis `_: Analysis for simulations produced with Model for Prediction Across Scales (MPAS) components and the Accelerated Climate Model for Energy (ACME). +- `OGGM `_: Open Global Glacier Model +- `Oocgcm `_: Analysis of large gridded geophysical datasets +- `Open Data Cube `_: Analysis toolkit of continental scale Earth Observation data from satellites. +- `Pangaea: `_: xarray extension for gridded land surface & weather model output). +- `Pangeo `_: A community effort for big data geoscience in the cloud. +- `PyGDX `_: Python 3 package for + accessing data stored in GAMS Data eXchange (GDX) files. Also uses a custom + subclass. +- `Regionmask `_: plotting and creation of masks of spatial regions +- `salem `_: Adds geolocalised subsetting, masking, and plotting operations to xarray's data structures via accessors. +- `Spyfit `_: FTIR spectroscopy of the atmosphere +- `windspharm `_: Spherical + harmonic wind analysis in Python. +- `wrf-python `_: A collection of diagnostic and interpolation routines for use with output of the Weather Research and Forecasting (WRF-ARW) Model. +- `xarray-simlab `_: xarray extension for computer model simulations. +- `xarray-topo `_: xarray extension for topographic analysis and modelling. +- `xbpch `_: xarray interface for bpch files. +- `xESMF `_: Universal Regridder for Geospatial Data. +- `xgcm `_: Extends the xarray data model to understand finite volume grid cells (common in General Circulation Models) and provides interpolation and difference operations for such grids. +- `xmitgcm `_: a python package for reading `MITgcm `_ binary MDS files into xarray data structures. +- `xshape `_: Tools for working with shapefiles, topographies, and polygons in xarray. + +Machine Learning +~~~~~~~~~~~~~~~~ +- `cesium `_: machine learning for time series analysis +- `Elm `_: Parallel machine learning on xarray data structures +- `sklearn-xarray (1) `_: Combines scikit-learn and xarray (1). +- `sklearn-xarray (2) `_: Combines scikit-learn and xarray (2). + +Extend xarray capabilities +~~~~~~~~~~~~~~~~~~~~~~~~~~ +- `Collocate `_: Collocate xarray trajectories in arbitrary physical dimensions +- `eofs `_: EOF analysis in Python. +- `xarray_extras `_: Advanced algorithms for xarray objects (e.g. intergrations/interpolations). +- `xrft `_: Fourier transforms for xarray data. +- `xr-scipy `_: A lightweight scipy wrapper for xarray. +- `X-regression `_: Multiple linear regression from Statsmodels library coupled with Xarray library. + +Visualization +~~~~~~~~~~~~~ +- `Datashader `_, `geoviews `_, `holoviews `_, : visualization packages for large data +- `psyplot `_: Interactive data visualization with python. + +Other +~~~~~ +- `ptsa `_: EEG Time Series Analysis +- `pycalphad `_: Computational Thermodynamics in Python + +More projects can be found at the `"xarray" Github topic `_. + How should I cite xarray? ------------------------- diff --git a/doc/internals.rst b/doc/internals.rst index e5e14896472..170e2d0b0cc 100644 --- a/doc/internals.rst +++ b/doc/internals.rst @@ -130,20 +130,3 @@ To help users keep things straight, please `let us know `_ if you plan to write a new accessor for an open source library. In the future, we will maintain a list of accessors and the libraries that implement them on this page. - -Here are several existing libraries that build functionality upon xarray. -They may be useful points of reference for your work: - -- `xgcm `_: General Circulation Model - Postprocessing. Uses subclassing and custom xarray backends. -- `PyGDX `_: Python 3 package for - accessing data stored in GAMS Data eXchange (GDX) files. Also uses a custom - subclass. -- `windspharm `_: Spherical - harmonic wind analysis in Python. -- `eofs `_: EOF analysis in Python. -- `salem `_: Adds geolocalised subsetting, - masking, and plotting operations to xarray's data structures via accessors. - -.. TODO: consider adding references to these projects somewhere more prominent -.. in the documentation? maybe the FAQ page? diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 49d39bacbf8..218df9b9707 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -31,6 +31,12 @@ What's New v0.10.4 (unreleased) -------------------- +Documentation +~~~~~~~~~~~~~ +- `FAQ `_ now lists projects that leverage xarray. + By `Deepak Cherian `_. + + Enhancements ~~~~~~~~~~~~ @@ -58,6 +64,8 @@ Enhancements This greatly boosts speed and allows chunking on the core dims. The function now requires dask >= 0.17.3 to work on dask-backed data (:issue:`2074`). By `Guido Imperiale `_. +- ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change the direction of the respective axes. + By `Deepak Cherian `_. Bug fixes ~~~~~~~~~ @@ -76,8 +84,6 @@ Bug fixes By `Stephan Hoyer `_. - ``plot.line()`` does not call ``autofmt_xdate()`` anymore. Instead it changes the rotation and horizontal alignment of labels without removing the x-axes of any other subplots in the figure (if any). By `Deepak Cherian `_. -- ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change the direction of the respective axes. - By `Deepak Cherian `_. - Colorbar limits are now determined by excluding ±Infs too. By `Deepak Cherian `_. From 218ad549a25fb30b836aabdfdda412450fdc9585 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 14 May 2018 22:06:37 +0100 Subject: [PATCH 108/282] xarray.dot to pass **kwargs to einsum (#2106) * Support for optimize, split_every, etc. * Avoid einsum params that aren't ubiquitously supported * Fix tests for einsum params * Stickler fix * Reinstate test for invalid parameters --- xarray/core/computation.py | 8 ++++---- xarray/tests/test_computation.py | 25 +++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 77a52ac055d..f6e22dfe6c1 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -952,6 +952,9 @@ def dot(*arrays, **kwargs): dims: str or tuple of strings, optional Which dimensions to sum over. If not speciified, then all the common dimensions are summed over. + **kwargs: dict + Additional keyword arguments passed to numpy.einsum or + dask.array.einsum Returns ------- @@ -976,9 +979,6 @@ def dot(*arrays, **kwargs): from .variable import Variable dims = kwargs.pop('dims', None) - if len(kwargs) > 0: - raise TypeError('Invalid keyward arguments {} are given'.format( - list(kwargs.keys()))) if any(not isinstance(arr, (Variable, DataArray)) for arr in arrays): raise TypeError('Only xr.DataArray and xr.Variable are supported.' @@ -1024,7 +1024,7 @@ def dot(*arrays, **kwargs): # subscripts should be passed to np.einsum as arg, not as kwargs. We need # to construct a partial function for apply_ufunc to work. - func = functools.partial(duck_array_ops.einsum, subscripts) + func = functools.partial(duck_array_ops.einsum, subscripts, **kwargs) result = apply_ufunc(func, *arrays, input_core_dims=input_core_dims, output_core_dims=output_core_dims, diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index c84ed17bfd3..c829453cc9d 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -1,5 +1,6 @@ import functools import operator +import pickle from collections import OrderedDict from distutils.version import LooseVersion @@ -842,13 +843,33 @@ def test_dot(use_dask): assert actual.dims == ('b', ) assert (actual.data == np.zeros(actual.shape)).all() - with pytest.raises(TypeError): - xr.dot(da_a, dims='a', invalid=None) + # Invalid cases + if not use_dask or LooseVersion(dask.__version__) > LooseVersion('0.17.4'): + with pytest.raises(TypeError): + xr.dot(da_a, dims='a', invalid=None) with pytest.raises(TypeError): xr.dot(da_a.to_dataset(name='da'), dims='a') with pytest.raises(TypeError): xr.dot(dims='a') + # einsum parameters + actual = xr.dot(da_a, da_b, dims=['b'], order='C') + assert (actual.data == np.einsum('ij,ijk->ik', a, b)).all() + assert actual.values.flags['C_CONTIGUOUS'] + assert not actual.values.flags['F_CONTIGUOUS'] + actual = xr.dot(da_a, da_b, dims=['b'], order='F') + assert (actual.data == np.einsum('ij,ijk->ik', a, b)).all() + # dask converts Fortran arrays to C order when merging the final array + if not use_dask: + assert not actual.values.flags['C_CONTIGUOUS'] + assert actual.values.flags['F_CONTIGUOUS'] + + # einsum has a constant string as of the first parameter, which makes + # it hard to pass to xarray.apply_ufunc. + # make sure dot() uses functools.partial(einsum, subscripts), which + # can be pickled, and not a lambda, which can't. + pickle.loads(pickle.dumps(xr.dot(da_a))) + def test_where(): cond = xr.DataArray([True, False], dims='x') From f4ef34f00902a55f65b82c998d29a4ab8f5b6bf0 Mon Sep 17 00:00:00 2001 From: Alex Hilson Date: Mon, 14 May 2018 23:53:21 +0100 Subject: [PATCH 109/282] Fix to_iris conversion issues (#2111) * TST: assert lazy array maintained by to_iris (#2046) * Add masked_invalid array op, resolves to_iris rechunking issue (#2046) * Fix dask_module in duck_array_ops.masked_invalid * Really fix it * Resolving to_iris dask array issues --- doc/whats-new.rst | 2 ++ xarray/convert.py | 10 ++-------- xarray/core/duck_array_ops.py | 4 ++++ xarray/tests/test_dataarray.py | 9 ++++----- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 218df9b9707..520e38bd80f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -86,6 +86,8 @@ Bug fixes By `Deepak Cherian `_. - Colorbar limits are now determined by excluding ±Infs too. By `Deepak Cherian `_. +- Fixed ``to_iris`` to maintain lazy dask array after conversion (:issue:`2046`). + By `Alex Hilson `_ and `Stephan Hoyer `_. .. _whats-new.0.10.3: diff --git a/xarray/convert.py b/xarray/convert.py index a6defd083bf..a3c99119306 100644 --- a/xarray/convert.py +++ b/xarray/convert.py @@ -6,6 +6,7 @@ from .coding.times import CFDatetimeCoder, CFTimedeltaCoder from .conventions import decode_cf +from .core import duck_array_ops from .core.dataarray import DataArray from .core.dtypes import get_fill_value from .core.pycompat import OrderedDict, range @@ -94,7 +95,6 @@ def to_iris(dataarray): # Iris not a hard dependency import iris from iris.fileformats.netcdf import parse_cell_methods - from xarray.core.pycompat import dask_array_type dim_coords = [] aux_coords = [] @@ -121,13 +121,7 @@ def to_iris(dataarray): args['cell_methods'] = \ parse_cell_methods(dataarray.attrs['cell_methods']) - # Create the right type of masked array (should be easier after #1769) - if isinstance(dataarray.data, dask_array_type): - from dask.array import ma as dask_ma - masked_data = dask_ma.masked_invalid(dataarray) - else: - masked_data = np.ma.masked_invalid(dataarray) - + masked_data = duck_array_ops.masked_invalid(dataarray.data) cube = iris.cube.Cube(masked_data, **args) return cube diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 3a2c123f87e..ef52b4890ef 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -101,6 +101,10 @@ def isnull(data): einsum = _dask_or_eager_func('einsum', array_args=slice(1, None), requires_dask='0.17.3') +masked_invalid = _dask_or_eager_func( + 'masked_invalid', eager_module=np.ma, + dask_module=getattr(dask_array, 'ma', None)) + def asarray(data): return data if isinstance(data, dask_array_type) else np.asarray(data) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index b16cb8ddcea..22bfecebe3c 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3038,12 +3038,11 @@ def test_to_and_from_iris_dask(self): roundtripped = DataArray.from_iris(actual) assert_identical(original, roundtripped) - # If the Iris version supports it then we should get a dask array back + # If the Iris version supports it then we should have a dask array + # at each stage of the conversion if hasattr(actual, 'core_data'): - pass - # TODO This currently fails due to the decoding loading - # the data (#1372) - # self.assertEqual(type(original.data), type(roundtripped.data)) + self.assertEqual(type(original.data), type(actual.core_data())) + self.assertEqual(type(original.data), type(roundtripped.data)) actual.remove_coord('time') auto_time_dimension = DataArray.from_iris(actual) From 9a48157b525d9e346e73f358a99ceb52717fd3ea Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Wed, 16 May 2018 01:39:22 +0900 Subject: [PATCH 110/282] Raise an Error if a coordinate with wrong size is assigned to a dataarray (#2124) * fix * Fix DataArrayCoordinates._update_coords * Update misleading comments --- doc/whats-new.rst | 5 ++++- xarray/core/coordinates.py | 11 +++++++++-- xarray/tests/test_dataarray.py | 7 +++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 520e38bd80f..98116aa2a95 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -70,8 +70,11 @@ Enhancements Bug fixes ~~~~~~~~~ +- Now raises an Error if a coordinate with wrong size is assigned to a + :py:class:`~xarray.DataArray`. (:issue:`2112`) + By `Keisuke Fujii `_. - Fixed a bug in `rolling` with bottleneck. Also, fixed a bug in rolling an - integer dask array. (:issue:`21133`) + integer dask array. (:issue:`2113`) By `Keisuke Fujii `_. - Fixed a bug where `keep_attrs=True` flag was neglected if :py:func:`apply_func` was used with :py:class:`Variable`. (:issue:`2114`) diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index 92d717b9f62..cb22c0b687b 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -9,10 +9,15 @@ from .merge import ( expand_and_merge_variables, merge_coords, merge_coords_for_inplace_math) from .pycompat import OrderedDict -from .utils import Frozen +from .utils import Frozen, ReprObject from .variable import Variable +# Used as the key corresponding to a DataArray's variable when converting +# arbitrary DataArray objects to datasets +_THIS_ARRAY = ReprObject('') + + class AbstractCoordinates(Mapping, formatting.ReprMixin): def __getitem__(self, key): raise NotImplementedError @@ -225,7 +230,9 @@ def __getitem__(self, key): def _update_coords(self, coords): from .dataset import calculate_dimensions - dims = calculate_dimensions(coords) + coords_plus_data = coords.copy() + coords_plus_data[_THIS_ARRAY] = self._data.variable + dims = calculate_dimensions(coords_plus_data) if not set(dims) <= set(self.dims): raise ValueError('cannot add coordinates with new dimensions to ' 'a DataArray') diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 22bfecebe3c..f2e076db78a 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1180,6 +1180,13 @@ def test_assign_coords(self): with raises_regex(ValueError, 'conflicting MultiIndex'): self.mda.assign_coords(level_1=range(4)) + # GH: 2112 + da = xr.DataArray([0, 1, 2], dims='x') + with pytest.raises(ValueError): + da['x'] = [0, 1, 2, 3] # size conflict + with pytest.raises(ValueError): + da.coords['x'] = [0, 1, 2, 3] # size conflict + def test_coords_alignment(self): lhs = DataArray([1, 2, 3], [('x', [0, 1, 2])]) rhs = DataArray([2, 3, 4], [('x', [1, 2, 3])]) From 3df3023c10ede416054bc8282ded858ba736424e Mon Sep 17 00:00:00 2001 From: chiaral Date: Tue, 15 May 2018 22:13:25 -0400 Subject: [PATCH 111/282] DOC: Added text to Assign values with Indexing (#2133) * DOC: Added text to Assign values with Indexing * DOC: Added Warning to xarray.DataArray.sel * DOC: Added Warning to xarray.DataArray.sel fixed length * DOC: Added info on whats-new --- doc/indexing.rst | 32 +++++++++++++++++++++++++++++++- doc/whats-new.rst | 2 ++ xarray/core/dataarray.py | 13 +++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/doc/indexing.rst b/doc/indexing.rst index 1f6ae006cf7..cec438dd2e4 100644 --- a/doc/indexing.rst +++ b/doc/indexing.rst @@ -398,7 +398,37 @@ These methods may and also be applied to ``Dataset`` objects Assigning values with indexing ------------------------------ -Vectorized indexing can be used to assign values to xarray object. +To select and assign values to a portion of a :py:meth:`~xarray.DataArray` you +can use indexing with ``.loc`` : + +.. ipython:: python + + ds = xr.tutorial.load_dataset('air_temperature') + + #add an empty 2D dataarray + ds['empty']= xr.full_like(ds.air.mean('time'),fill_value=0) + + #modify one grid point using loc() + ds['empty'].loc[dict(lon=260, lat=30)] = 100 + + #modify a 2D region using loc() + lc = ds.coords['lon'] + la = ds.coords['lat'] + ds['empty'].loc[dict(lon=lc[(lc>220)&(lc<260)], lat=la[(la>20)&(la<60)])] = 100 + +or :py:meth:`~xarray.where`: + +.. ipython:: python + + #modify one grid point using xr.where() + ds['empty'] = xr.where((ds.coords['lat']==20)&(ds.coords['lon']==260), 100, ds['empty']) + + #or modify a 2D region using xr.where() + mask = (ds.coords['lat']>20)&(ds.coords['lat']<60)&(ds.coords['lon']>220)&(ds.coords['lon']<260) + ds['empty'] = xr.where(mask, 100, ds['empty']) + + +Vectorized indexing can also be used to assign values to xarray object. .. ipython:: python diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 98116aa2a95..0d9e75ba940 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -35,6 +35,8 @@ Documentation ~~~~~~~~~~~~~ - `FAQ `_ now lists projects that leverage xarray. By `Deepak Cherian `_. +- `Assigning values with indexing `_ now includes examples on how to select and assign values to a :py:class:`~xarray.DataArray`. + By `Chiara Lepore `_. Enhancements diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 1ceaced5961..fc7091dad85 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -758,10 +758,23 @@ def sel(self, method=None, tolerance=None, drop=False, **indexers): """Return a new DataArray whose dataset is given by selecting index labels along the specified dimension(s). + .. warning:: + + Do not try to assign values when using any of the indexing methods + ``isel`` or ``sel``:: + + da = xr.DataArray([0, 1, 2, 3], dims=['x']) + # DO NOT do this + da.isel(x=[0, 1, 2])[1] = -1 + + Assigning values with the chained indexing using ``.sel`` or + ``.isel`` fails silently. + See Also -------- Dataset.sel DataArray.isel + """ ds = self._to_temp_dataset().sel(drop=drop, method=method, tolerance=tolerance, **indexers) From 9f58d509a432f18d9ceb69bdc0808f2cb9b77f6c Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Tue, 15 May 2018 21:02:46 -0700 Subject: [PATCH 112/282] Fix test suite with pandas 0.23 (#2136) * Fix test suite with pandas 0.23 * Disable -OO check --- .travis.yml | 4 +++- xarray/tests/test_dataset.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e375f6fb063..bd53edb0029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -101,7 +101,9 @@ install: - python xarray/util/print_versions.py script: - - python -OO -c "import xarray" + # TODO: restore this check once the upstream pandas issue is fixed: + # https://github.com/pandas-dev/pandas/issues/21071 + # - python -OO -c "import xarray" - if [[ "$CONDA_ENV" == "docs" ]]; then conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc; sphinx-build -n -j auto -b html -d _build/doctrees doc _build/html; diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index b99f7ea1eec..3335a55e4ab 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1848,7 +1848,9 @@ def test_drop_index_labels(self): expected = data.isel(x=slice(0, 0)) assert_identical(expected, actual) - with pytest.raises(ValueError): + # This exception raised by pandas changed from ValueError -> KeyError + # in pandas 0.23. + with pytest.raises((ValueError, KeyError)): # not contained in axis data.drop(['c'], dim='x') From 8ef194f2e6f2e68f1f818606d6362ddfe801df1e Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 16 May 2018 11:05:02 -0400 Subject: [PATCH 113/282] WIP: Compute==False for to_zarr and to_netcdf (#1811) * move backend append logic to the prepare_variable methods * deprecate variables/dimensions/attrs properties on AbstractWritableDataStore * warnings instead of errors for backend properties * use attrs.update when setting zarr attributes * more performance improvements to attributes in zarr backend * fix typo * new set_dimensions method for writable data stores * more fixes for zarr * more tests for zarr and remove append logic for zarr * more tests for zarr and remove append logic for zarr * a few more tweaks to zarr attrs * Add encode methods to writable data stores, fixes for Zarr tests * fix for InMemoryDataStore * fix for unlimited dimensions Scipy Datastores * another patch for scipy * whatsnew * initial commit returning dask futures from to_netcdf and to_zarr methods * ordereddict * address some of rabernats comments, in particular, this commit removes the _DIMENSION_KEY from the zarr_group.attrs * stop skipping zero-dim zarr tests * update minimum zarr version for tests * cleanup a bit before adding tests * tempoary checkin * cleanup implementation of compute=False for to_foo functions, still needs additional tests * docs and more tests, failing tests on h5netcdf backend only * skip h5netcdf/netcdf4 tests in certain places * remove spurious returns * finalize stores when compute=False * more docs, skip h5netcdf netcdf tests, raise informative error for h5netcdf and scipy * cleanup whats-new * reorg dask task graph when using compute=False and save_mfdataset * move compute_false tests to DaskTests class * small doc/style fixes * save api.py --- doc/dask.rst | 15 ++++++++++ doc/whats-new.rst | 19 ++++++++---- xarray/backends/api.py | 42 ++++++++++++++++++++------ xarray/backends/common.py | 12 +++++--- xarray/backends/h5netcdf_.py | 7 +++-- xarray/backends/netCDF4_.py | 4 +-- xarray/backends/scipy_.py | 7 +++-- xarray/backends/zarr.py | 4 +-- xarray/core/dataset.py | 20 +++++++++---- xarray/tests/test_backends.py | 56 +++++++++++++++++++++++++++++++---- 10 files changed, 147 insertions(+), 39 deletions(-) diff --git a/doc/dask.rst b/doc/dask.rst index 8fc0f655023..2d4beea4f70 100644 --- a/doc/dask.rst +++ b/doc/dask.rst @@ -100,6 +100,21 @@ Once you've manipulated a dask array, you can still write a dataset too big to fit into memory back to disk by using :py:meth:`~xarray.Dataset.to_netcdf` in the usual way. +.. ipython:: python + + ds.to_netcdf('manipulated-example-data.nc') + +By setting the ``compute`` argument to ``False``, :py:meth:`~xarray.Dataset.to_netcdf` +will return a dask delayed object that can be computed later. + +.. ipython:: python + + from dask.diagnostics import ProgressBar + # or distributed.progress when using the distributed scheduler + delayed_obj = ds.to_netcdf('manipulated-example-data.nc', compute=False) + with ProgressBar(): + results = delayed_obj.compute() + .. note:: When using dask's distributed scheduler to write NETCDF4 files, diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0d9e75ba940..1b696c4486d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -69,6 +69,19 @@ Enhancements - ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change the direction of the respective axes. By `Deepak Cherian `_. +- Added the ``parallel`` option to :py:func:`open_mfdataset`. This option uses + ``dask.delayed`` to parallelize the open and preprocessing steps within + ``open_mfdataset``. This is expected to provide performance improvements when + opening many files, particularly when used in conjunction with dask's + multiprocessing or distributed schedulers (:issue:`1981`). + By `Joe Hamman `_. + +- New ``compute`` option in :py:meth:`~xarray.Dataset.to_netcdf`, + :py:meth:`~xarray.Dataset.to_zarr`, and :py:func:`~xarray.save_mfdataset` to + allow for the lazy computation of netCDF and zarr stores. This feature is + currently only supported by the netCDF4 and zarr backends. (:issue:`1784`). + By `Joe Hamman `_. + Bug fixes ~~~~~~~~~ @@ -104,12 +117,6 @@ The minor release includes a number of bug-fixes and backwards compatible enhanc Enhancements ~~~~~~~~~~~~ -- Added the ``parallel`` option to :py:func:`open_mfdataset`. This option uses - ``dask.delayed`` to parallelize the open and preprocessing steps within - ``open_mfdataset``. This is expected to provide performance improvements when - opening many files, particularly when used in conjunction with dask's - multiprocessing or distributed schedulers (:issue:`1981`). - By `Joe Hamman `_. - :py:meth:`~xarray.DataArray.isin` and :py:meth:`~xarray.Dataset.isin` methods, which test each value in the array for whether it is contained in the supplied list, returning a bool array. See :ref:`selecting values with isin` diff --git a/xarray/backends/api.py b/xarray/backends/api.py index b8cfa3c926a..dec63a85d6e 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -144,6 +144,13 @@ def _get_lock(engine, scheduler, format, path_or_file): return lock +def _finalize_store(write, store): + """ Finalize this store by explicitly syncing and closing""" + del write # ensure writing is done first + store.sync() + store.close() + + def open_dataset(filename_or_obj, group=None, decode_cf=True, mask_and_scale=True, decode_times=True, autoclose=False, concat_characters=True, decode_coords=True, engine=None, @@ -620,7 +627,8 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, - engine=None, writer=None, encoding=None, unlimited_dims=None): + engine=None, writer=None, encoding=None, unlimited_dims=None, + compute=True): """This function creates an appropriate datastore for writing a dataset to disk as a netCDF file @@ -680,19 +688,22 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, unlimited_dims = dataset.encoding.get('unlimited_dims', None) try: dataset.dump_to_store(store, sync=sync, encoding=encoding, - unlimited_dims=unlimited_dims) + unlimited_dims=unlimited_dims, compute=compute) if path_or_file is None: return target.getvalue() finally: if sync and isinstance(path_or_file, basestring): store.close() + if not compute: + import dask + return dask.delayed(_finalize_store)(store.delayed_store, store) + if not sync: return store - def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, - engine=None): + engine=None, compute=True): """Write multiple datasets to disk as netCDF files simultaneously. This function is intended for use with datasets consisting of dask.array @@ -742,6 +753,9 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, default engine is chosen based on available dependencies, with a preference for 'netcdf4' if writing to a file on disk. See `Dataset.to_netcdf` for additional information. + compute: boolean + If true compute immediately, otherwise return a + ``dask.delayed.Delayed`` object that can be computed later. Examples -------- @@ -769,11 +783,17 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, 'datasets, paths and groups arguments to ' 'save_mfdataset') - writer = ArrayWriter() - stores = [to_netcdf(ds, path, mode, format, group, engine, writer) + writer = ArrayWriter() if compute else None + stores = [to_netcdf(ds, path, mode, format, group, engine, writer, + compute=compute) for ds, path, group in zip(datasets, paths, groups)] + + if not compute: + import dask + return dask.delayed(stores) + try: - writer.sync() + delayed = writer.sync(compute=compute) for store in stores: store.sync() finally: @@ -782,7 +802,7 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, def to_zarr(dataset, store=None, mode='w-', synchronizer=None, group=None, - encoding=None): + encoding=None, compute=True): """This function creates an appropriate datastore for writing a dataset to a zarr ztore @@ -803,5 +823,9 @@ def to_zarr(dataset, store=None, mode='w-', synchronizer=None, group=None, # I think zarr stores should always be sync'd immediately # TODO: figure out how to properly handle unlimited_dims - dataset.dump_to_store(store, sync=True, encoding=encoding) + dataset.dump_to_store(store, sync=True, encoding=encoding, compute=compute) + + if not compute: + import dask + return dask.delayed(_finalize_store)(store.delayed_store, store) return store diff --git a/xarray/backends/common.py b/xarray/backends/common.py index 7d8aa8446a2..2961838e85f 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -264,12 +264,15 @@ def add(self, source, target): else: target[...] = source - def sync(self): + def sync(self, compute=True): if self.sources: import dask.array as da - da.store(self.sources, self.targets, lock=self.lock) + delayed_store = da.store(self.sources, self.targets, + lock=self.lock, compute=compute, + flush=True) self.sources = [] self.targets = [] + return delayed_store class AbstractWritableDataStore(AbstractDataStore): @@ -277,6 +280,7 @@ def __init__(self, writer=None, lock=HDF5_LOCK): if writer is None: writer = ArrayWriter(lock=lock) self.writer = writer + self.delayed_store = None def encode(self, variables, attributes): """ @@ -318,11 +322,11 @@ def set_attribute(self, k, v): # pragma: no cover def set_variable(self, k, v): # pragma: no cover raise NotImplementedError - def sync(self): + def sync(self, compute=True): if self._isopen and self._autoclose: # datastore will be reopened during write self.close() - self.writer.sync() + self.delayed_store = self.writer.sync(compute=compute) def store_dataset(self, dataset): """ diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index d34fa2d9267..f9e2b3dece1 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -212,9 +212,12 @@ def prepare_variable(self, name, variable, check_encoding=False, return target, variable.data - def sync(self): + def sync(self, compute=True): + if not compute: + raise NotImplementedError( + 'compute=False is not supported for the h5netcdf backend yet') with self.ensure_open(autoclose=True): - super(H5NetCDFStore, self).sync() + super(H5NetCDFStore, self).sync(compute=compute) self.ds.sync() def close(self): diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index a0f6cbcdd33..14061a0fb08 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -439,9 +439,9 @@ def prepare_variable(self, name, variable, check_encoding=False, return target, variable.data - def sync(self): + def sync(self, compute=True): with self.ensure_open(autoclose=True): - super(NetCDF4DataStore, self).sync() + super(NetCDF4DataStore, self).sync(compute=compute) self.ds.sync() def close(self): diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index ee2c0fbf106..cd84431f6b7 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -219,9 +219,12 @@ def prepare_variable(self, name, variable, check_encoding=False, return target, data - def sync(self): + def sync(self, compute=True): + if not compute: + raise NotImplementedError( + 'compute=False is not supported for the scipy backend yet') with self.ensure_open(autoclose=True): - super(ScipyDataStore, self).sync() + super(ScipyDataStore, self).sync(compute=compute) self.ds.flush() def close(self): diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 83dcbd9a172..343690eaabd 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -342,8 +342,8 @@ def store(self, variables, attributes, *args, **kwargs): AbstractWritableDataStore.store(self, variables, attributes, *args, **kwargs) - def sync(self): - self.writer.sync() + def sync(self, compute=True): + self.delayed_store = self.writer.sync(compute=compute) def open_zarr(store, group=None, synchronizer=None, auto_chunk=True, diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index bdb2bf86990..a9ec8c16866 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1055,7 +1055,7 @@ def reset_coords(self, names=None, drop=False, inplace=False): return obj def dump_to_store(self, store, encoder=None, sync=True, encoding=None, - unlimited_dims=None): + unlimited_dims=None, compute=True): """Store dataset contents to a backends.*DataStore object.""" if encoding is None: encoding = {} @@ -1074,10 +1074,11 @@ def dump_to_store(self, store, encoder=None, sync=True, encoding=None, store.store(variables, attrs, check_encoding, unlimited_dims=unlimited_dims) if sync: - store.sync() + store.sync(compute=compute) def to_netcdf(self, path=None, mode='w', format=None, group=None, - engine=None, encoding=None, unlimited_dims=None): + engine=None, encoding=None, unlimited_dims=None, + compute=True): """Write dataset contents to a netCDF file. Parameters @@ -1136,16 +1137,20 @@ def to_netcdf(self, path=None, mode='w', format=None, group=None, By default, no dimensions are treated as unlimited dimensions. Note that unlimited_dims may also be set via ``dataset.encoding['unlimited_dims']``. + compute: boolean + If true compute immediately, otherwise return a + ``dask.delayed.Delayed`` object that can be computed later. """ if encoding is None: encoding = {} from ..backends.api import to_netcdf return to_netcdf(self, path, mode, format=format, group=group, engine=engine, encoding=encoding, - unlimited_dims=unlimited_dims) + unlimited_dims=unlimited_dims, + compute=compute) def to_zarr(self, store=None, mode='w-', synchronizer=None, group=None, - encoding=None): + encoding=None, compute=True): """Write dataset contents to a zarr group. .. note:: Experimental @@ -1167,6 +1172,9 @@ def to_zarr(self, store=None, mode='w-', synchronizer=None, group=None, Nested dictionary with variable names as keys and dictionaries of variable specific encodings as values, e.g., ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1,}, ...}`` + compute: boolean + If true compute immediately, otherwise return a + ``dask.delayed.Delayed`` object that can be computed later. """ if encoding is None: encoding = {} @@ -1176,7 +1184,7 @@ def to_zarr(self, store=None, mode='w-', synchronizer=None, group=None, "and 'w-'.") from ..backends.api import to_zarr return to_zarr(self, store=store, mode=mode, synchronizer=synchronizer, - group=group, encoding=encoding) + group=group, encoding=encoding, compute=compute) def __unicode__(self): return formatting.dataset_repr(self) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 2d4e5c0f261..95d92cd8b8a 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -159,8 +159,8 @@ def roundtrip_append(self, data, save_kwargs={}, open_kwargs={}, # The save/open methods may be overwritten below def save(self, dataset, path, **kwargs): - dataset.to_netcdf(path, engine=self.engine, format=self.file_format, - **kwargs) + return dataset.to_netcdf(path, engine=self.engine, + format=self.file_format, **kwargs) @contextlib.contextmanager def open(self, path, **kwargs): @@ -709,7 +709,7 @@ def test_roundtrip_endian(self): # should still pass though. assert_identical(ds, actual) - if isinstance(self, NetCDF4DataTest): + if self.engine == 'netcdf4': ds['z'].encoding['endian'] = 'big' with pytest.raises(NotImplementedError): with self.roundtrip(ds) as actual: @@ -902,7 +902,8 @@ def test_open_group(self): open_dataset(tmp_file, group=(1, 2, 3)) def test_open_subgroup(self): - # Create a netCDF file with a dataset within a group within a group + # Create a netCDF file with a dataset stored within a group within a + # group with create_tmp_file() as tmp_file: rootgrp = nc4.Dataset(tmp_file, 'w') foogrp = rootgrp.createGroup('foo') @@ -1232,7 +1233,7 @@ def create_store(self): yield backends.ZarrStore.open_group(store_target, mode='w') def save(self, dataset, store_target, **kwargs): - dataset.to_zarr(store=store_target, **kwargs) + return dataset.to_zarr(store=store_target, **kwargs) @contextlib.contextmanager def open(self, store_target, **kwargs): @@ -1419,6 +1420,19 @@ def test_append_overwrite_values(self): def test_append_with_invalid_dim_raises(self): super(CFEncodedDataTest, self).test_append_with_invalid_dim_raises() + def test_to_zarr_compute_false_roundtrip(self): + from dask.delayed import Delayed + + original = create_test_data().chunk() + + with self.create_zarr_target() as store: + delayed_obj = self.save(original, store, compute=False) + assert isinstance(delayed_obj, Delayed) + delayed_obj.compute() + + with self.open(store) as actual: + assert_identical(original, actual) + @requires_zarr class ZarrDictStoreTest(BaseZarrTest, TestCase): @@ -2227,6 +2241,36 @@ def test_dataarray_compute(self): self.assertTrue(computed._in_memory) assert_allclose(actual, computed, decode_bytes=False) + def test_to_netcdf_compute_false_roundtrip(self): + from dask.delayed import Delayed + + original = create_test_data().chunk() + + with create_tmp_file() as tmp_file: + # dataset, path, **kwargs): + delayed_obj = self.save(original, tmp_file, compute=False) + assert isinstance(delayed_obj, Delayed) + delayed_obj.compute() + + with self.open(tmp_file) as actual: + assert_identical(original, actual) + + def test_save_mfdataset_compute_false_roundtrip(self): + from dask.delayed import Delayed + + original = Dataset({'foo': ('x', np.random.randn(10))}).chunk() + datasets = [original.isel(x=slice(5)), + original.isel(x=slice(5, 10))] + with create_tmp_file() as tmp1: + with create_tmp_file() as tmp2: + delayed_obj = save_mfdataset(datasets, [tmp1, tmp2], + engine=self.engine, compute=False) + assert isinstance(delayed_obj, Delayed) + delayed_obj.compute() + with open_mfdataset([tmp1, tmp2], + autoclose=self.autoclose) as actual: + assert_identical(actual, original) + class DaskTestAutocloseTrue(DaskTest): autoclose = True @@ -2348,7 +2392,7 @@ def open(self, path, **kwargs): yield ds def save(self, dataset, path, **kwargs): - dataset.to_netcdf(path, engine='scipy', **kwargs) + return dataset.to_netcdf(path, engine='scipy', **kwargs) def test_weakrefs(self): example = Dataset({'foo': ('x', np.arange(5.0))}) From 4972dfd84d4e7ed31875b4257492ca84939eda4a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 16 May 2018 15:47:59 -0400 Subject: [PATCH 114/282] expose CFTimeIndex to public API (#2141) * expose CFTimeIndex to public API * more docs --- doc/api.rst | 7 +++++++ xarray/__init__.py | 2 ++ xarray/coding/cftimeindex.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/doc/api.rst b/doc/api.rst index bce4e0d1c8e..ff708dc4c1b 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -522,6 +522,13 @@ GroupByObjects core.groupby.DatasetGroupBy.apply core.groupby.DatasetGroupBy.reduce +Custom Indexes +============== +.. autosummary:: + :toctree: generated/ + + CFTimeIndex + Plotting ======== diff --git a/xarray/__init__.py b/xarray/__init__.py index 1a2bf3fe283..94e8029edbb 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -22,6 +22,8 @@ from .conventions import decode_cf, SerializationWarning +from .coding.cftimeindex import CFTimeIndex + try: from .version import version as __version__ except ImportError: # pragma: no cover diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index fb51ace5d69..5fca14ddbb1 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -135,6 +135,10 @@ def assert_all_valid_date_type(data): class CFTimeIndex(pd.Index): + """Custom Index for working with CF calendars and dates + + All elements of a CFTimeIndex must be cftime.datetime objects. + """ year = _field_accessor('year', 'The year of the datetime') month = _field_accessor('month', 'The month of the datetime') day = _field_accessor('day', 'The days of the datetime') From 954b8d0ce72eadba812821a2e64ae0ef4ceb2767 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 16 May 2018 18:11:00 -0700 Subject: [PATCH 115/282] Doc updates for 0.10.4 release (#2138) * Doc updates for 0.10.4 release * Fix to_netcdf() with engine=h5netcdf entry in whatsnew --- doc/api.rst | 38 ++++++++++++++++++++++++++++++-------- doc/faq.rst | 14 +++++++++----- doc/whats-new.rst | 41 ++++++++++++++++++++++++----------------- 3 files changed, 63 insertions(+), 30 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index ff708dc4c1b..a528496bb6a 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -496,6 +496,19 @@ DataArray methods DataArray.load DataArray.chunk +GroupBy objects +=============== + +.. autosummary:: + :toctree: generated/ + + core.groupby.DataArrayGroupBy + core.groupby.DataArrayGroupBy.apply + core.groupby.DataArrayGroupBy.reduce + core.groupby.DatasetGroupBy + core.groupby.DatasetGroupBy.apply + core.groupby.DatasetGroupBy.reduce + Rolling objects =============== @@ -509,18 +522,27 @@ Rolling objects core.rolling.DatasetRolling.construct core.rolling.DatasetRolling.reduce -GroupByObjects -============== +Resample objects +================ + +Resample objects also implement the GroupBy interface +(methods like ``apply()``, ``reduce()``, ``mean()``, ``sum()``, etc.). .. autosummary:: :toctree: generated/ - core.groupby.DataArrayGroupBy - core.groupby.DataArrayGroupBy.apply - core.groupby.DataArrayGroupBy.reduce - core.groupby.DatasetGroupBy - core.groupby.DatasetGroupBy.apply - core.groupby.DatasetGroupBy.reduce + core.resample.DataArrayResample + core.resample.DataArrayResample.asfreq + core.resample.DataArrayResample.backfill + core.resample.DataArrayResample.interpolate + core.resample.DataArrayResample.nearest + core.resample.DataArrayResample.pad + core.resample.DatasetResample + core.resample.DatasetResample.asfreq + core.resample.DatasetResample.backfill + core.resample.DatasetResample.interpolate + core.resample.DatasetResample.nearest + core.resample.DatasetResample.pad Custom Indexes ============== diff --git a/doc/faq.rst b/doc/faq.rst index 46f1e20f4e8..360cdb50791 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -1,3 +1,5 @@ +.. _faq: + Frequently Asked Questions ========================== @@ -129,8 +131,8 @@ What other netCDF related Python libraries should I know about? `netCDF4-python`__ provides a lower level interface for working with netCDF and OpenDAP datasets in Python. We use netCDF4-python internally in xarray, and have contributed a number of improvements and fixes upstream. xarray -does not yet support all of netCDF4-python's features, such as writing to -netCDF groups or modifying files on-disk. +does not yet support all of netCDF4-python's features, such as modifying files +on-disk. __ https://github.com/Unidata/netcdf4-python @@ -153,10 +155,12 @@ __ http://drclimate.wordpress.com/2014/01/02/a-beginners-guide-to-scripting-with We think the design decisions we have made for xarray (namely, basing it on pandas) make it a faster and more flexible data analysis tool. That said, Iris -and CDAT have some great domain specific functionality, and we would love to -have support for converting their native objects to and from xarray (see -:issue:`37` and :issue:`133`) +and CDAT have some great domain specific functionality, and xarray includes +methods for converting back and forth between xarray and these libraries. See +:py:meth:`~xarray.DataArray.to_iris` and :py:meth:`~xarray.DataArray.to_cdms2` +for more details. +.. _faq.other_projects: What other projects leverage xarray? ------------------------------------ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 1b696c4486d..fbe7fc5edca 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -31,33 +31,37 @@ What's New v0.10.4 (unreleased) -------------------- +The minor release includes a number of bug-fixes and backwards compatible +enhancements. A highlight is ``CFTimeIndex``, which offers support for +non-standard calendars used in climate modeling. + Documentation ~~~~~~~~~~~~~ -- `FAQ `_ now lists projects that leverage xarray. + +- New FAQ entry, :ref:`faq.other_projects`. By `Deepak Cherian `_. -- `Assigning values with indexing `_ now includes examples on how to select and assign values to a :py:class:`~xarray.DataArray`. +- :ref:`assigning_values` now includes examples on how to select and assign + values to a :py:class:`~xarray.DataArray` with ``.loc``. By `Chiara Lepore `_. - Enhancements ~~~~~~~~~~~~ -- Slight modification in `rolling` with dask.array and bottleneck. Also, fixed a bug in rolling an - integer dask array. - By `Keisuke Fujii `_. - Add an option for using a ``CFTimeIndex`` for indexing times with non-standard calendars and/or outside the Timestamp-valid range; this index enables a subset of the functionality of a standard - ``pandas.DatetimeIndex`` (:issue:`789`, :issue:`1084`, :issue:`1252`). + ``pandas.DatetimeIndex``. + See :ref:`CFTimeIndex` for full details. + (:issue:`789`, :issue:`1084`, :issue:`1252`) By `Spencer Clark `_ with help from `Stephan Hoyer `_. - Allow for serialization of ``cftime.datetime`` objects (:issue:`789`, :issue:`1084`, :issue:`2008`, :issue:`1252`) using the standalone ``cftime`` - library. By `Spencer Clark - `_. + library. + By `Spencer Clark `_. - Support writing lists of strings as netCDF attributes (:issue:`2044`). By `Dan Nowacki `_. -- :py:meth:`~xarray.Dataset.to_netcdf(engine='h5netcdf')` now accepts h5py +- :py:meth:`~xarray.Dataset.to_netcdf` with ``engine='h5netcdf'`` now accepts h5py encoding settings ``compression`` and ``compression_opts``, along with the NetCDF4-Python style settings ``gzip=True`` and ``complevel``. This allows using any compression plugin installed in hdf5, e.g. LZF @@ -66,7 +70,8 @@ Enhancements This greatly boosts speed and allows chunking on the core dims. The function now requires dask >= 0.17.3 to work on dask-backed data (:issue:`2074`). By `Guido Imperiale `_. -- ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change the direction of the respective axes. +- ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change + the direction of the respective axes. By `Deepak Cherian `_. - Added the ``parallel`` option to :py:func:`open_mfdataset`. This option uses @@ -85,14 +90,14 @@ Enhancements Bug fixes ~~~~~~~~~ -- Now raises an Error if a coordinate with wrong size is assigned to a - :py:class:`~xarray.DataArray`. (:issue:`2112`) +- ``ValueError`` is raised when coordinates with the wrong size are assigned to + a :py:class:`DataArray`. (:issue:`2112`) By `Keisuke Fujii `_. -- Fixed a bug in `rolling` with bottleneck. Also, fixed a bug in rolling an - integer dask array. (:issue:`2113`) +- Fixed a bug in :py:meth:`~xarary.DatasArray.rolling` with bottleneck. Also, + fixed a bug in rolling an integer dask array. (:issue:`2113`) By `Keisuke Fujii `_. - Fixed a bug where `keep_attrs=True` flag was neglected if - :py:func:`apply_func` was used with :py:class:`Variable`. (:issue:`2114`) + :py:func:`apply_ufunc` was used with :py:class:`Variable`. (:issue:`2114`) By `Keisuke Fujii `_. - When assigning a :py:class:`DataArray` to :py:class:`Dataset`, any conflicted non-dimensional coordinates of the DataArray are now dropped. @@ -100,7 +105,9 @@ Bug fixes By `Keisuke Fujii `_. - Better error handling in ``open_mfdataset`` (:issue:`2077`). By `Stephan Hoyer `_. -- ``plot.line()`` does not call ``autofmt_xdate()`` anymore. Instead it changes the rotation and horizontal alignment of labels without removing the x-axes of any other subplots in the figure (if any). +- ``plot.line()`` does not call ``autofmt_xdate()`` anymore. Instead it changes + the rotation and horizontal alignment of labels without removing the x-axes of + any other subplots in the figure (if any). By `Deepak Cherian `_. - Colorbar limits are now determined by excluding ±Infs too. By `Deepak Cherian `_. From 5d7304ea49dc04d7ce0d11947437fb0ad1fbd001 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 16 May 2018 18:12:15 -0700 Subject: [PATCH 116/282] Release v0.10.4 --- doc/whats-new.rst | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fbe7fc5edca..9a12f07a914 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -28,8 +28,8 @@ What's New .. _whats-new.0.10.4: -v0.10.4 (unreleased) --------------------- +v0.10.4 (May 16, 2018) +---------------------- The minor release includes a number of bug-fixes and backwards compatible enhancements. A highlight is ``CFTimeIndex``, which offers support for diff --git a/setup.py b/setup.py index c7c02c90e2f..c5e2ba831b7 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ MAJOR = 0 MINOR = 10 -MICRO = 3 -ISRELEASED = False +MICRO = 4 +ISRELEASED = True VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From 008c2c8e7544b0d8ea4e2fecde5625afabe6ea63 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 16 May 2018 18:17:27 -0700 Subject: [PATCH 117/282] Revert to dev version for 0.10.5 --- doc/whats-new.rst | 15 +++++++++++++++ setup.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 9a12f07a914..48abb892350 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -26,6 +26,21 @@ What's New - `Tips on porting to Python 3 `__ +.. _whats-new.0.10.5: + +v0.10.5 (unreleased) +-------------------- + +Documentation +~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + + .. _whats-new.0.10.4: v0.10.4 (May 16, 2018) diff --git a/setup.py b/setup.py index c5e2ba831b7..b5130958c00 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ MAJOR = 0 MINOR = 10 MICRO = 4 -ISRELEASED = True +ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) QUALIFIER = '' From 0a766b38de1d11f4c3110b267db72cb73e238d07 Mon Sep 17 00:00:00 2001 From: Katrin Leinweber <9948149+katrinleinweber@users.noreply.github.com> Date: Thu, 17 May 2018 15:16:47 +0200 Subject: [PATCH 118/282] Hyperlink DOI against preferred resolver (#2147) --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index 360cdb50791..9d763f1c15f 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -266,5 +266,5 @@ would certainly appreciate it. We recommend two citations. month = aug, year = 2016, doi = {10.5281/zenodo.59499}, - url = {http://dx.doi.org/10.5281/zenodo.59499} + url = {https://doi.org/10.5281/zenodo.59499} } From 7bab27cc637a60bff2b510d4f4a419c9754eeaa3 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Thu, 17 May 2018 18:55:30 +0200 Subject: [PATCH 119/282] Add favicon to docs? (#2146) --- doc/_static/favicon.ico | Bin 0 -> 4286 bytes doc/conf.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 doc/_static/favicon.ico diff --git a/doc/_static/favicon.ico b/doc/_static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..a1536e3ef76bf14b1cf5357b3cc3b9e63de94b80 GIT binary patch literal 4286 zcmds4dsLHG62CDV)UCRp*0!rqtV%$vJQVVPibl}tYFoE<-DoMvtx8;nO2NVV2nB7Ko^^fgZ%%m*CeaiMrjk>2PD2UA2VjaZ)X=du%>?& z9JpH$8no9dHY#KOk-Zu6gWsgdIYy<+%=9n!u4#Gjg>o(LGYP&BwJhaBMW%|Y)yrl z*?LYRE_?g$=FD=deJP;Y5BTq!g_ zou(VAFowF4Zq(hy?b^cUP*Zpx0(=zUFR6g_;!=zu?aJ8SO%X9QSg_BlUa-(7z_vaG z_W4CMn_z1K5nUJq z#^HDQ!dsIgR?Tv~*6t}afK2ifB;pR3F(WU@#x`Pn%2AClEwS7{^=&BY;T+e?x_K;= z2g*6&wPNfCLR@!A-2p+qO7N4EgTJ%_W2lC89>riiL9&3YyVT7>9s%3djRRxY+jjR4cJZa@xQT`_Er;5dvkE5P)?EkX7|rzwYrz%=vQ`_6%5fr!j`$d%CeM`}VrK z*0so_ov8l_cwrq}T`ub!r(Y1AM1G$CTx0&LDF0Y{;y300N`ao|fqH`KP@dZc8+`LI z?^@)q;$ywItj~G^^83|3XInaeZSTbx48Vfhb>m!P_PD#&w|L47AQe9X>H}Bj>SmGs zX;JF2GGThs#jd)NUTCcN5o+<=HI~1C+G6ZSZ7*szp2s>(4>V|BU@u~gO7Q&6&x8PJ zCC|SxhRVLN6C|j=vYYFt9@vI4Y-!V3yT$CC?NW>RC;d-BhIMdtxzg<9lp_k?a8S5? zQ(Wh%0~K)cP#K)~s|HRVDu?g(Yv9x&EhK$g0;djY(JqD4vFMK}gX1wOSo>Z&eBq&i z4W5^H3>ZtF70-W_frUOc=AP;h(xJ0h~Bu=N(ihkQSPxF#|D4n(X}z!x$Vte2GX7_b-nULWn?!^E5A z%)kC$EaExVpa-S~&6^cm*Alk+=HKQCIcbe>?zl6lKN{$Tm> zuLF}|N5C0~-k6S<0=qVx?Z-6O8JNoLZ-P?cE5DQQZIBisR%K%h1$Yih=?wIZeQk=I z&NQ37VG#>#`h}l?;CuI&B-Dr8_S24s&YpGS??(NXnjIX{L`r3nP^CKYqNAe}9zVGV z_wE6CE&x)Kf_mP16C81@9^K-i~QutuVU@uTcb{6;e0 zL!YoUg1$!P$&|Fd3i~O_V!}9f4>}BTh)kKxQiO^-77=R ze2#5x8bmGTqZq#bz8DT1@PWicF?_Yn6T;RU`{_^KJ8p~|HcrABrkLr3?2VaX=J25> z`7JH^$39kZ&YwRh%+5Y+Xl~8{olb>&XK_x-gHD$ZO-&aN^Dr;;JRkFt z1NHR^*uP&6@$nKkaY6=>JM7yREn|+f@t5;c`2P3~HX`1Ij14mxmX`eWXEIr4W}Gl5 zC*D9hTU(2G-U*4brA3YLWb?WZH|2)bH8kYGp+i0p7bk{<1SuRp?g^2R&iWA}EGC&Z z?w<(QFe@DI{Mv7_v$NxcO64&_Yil92x0fQXiX-(vTU!Z_hmg3b2DGlJsQ~rQ=lYKx zl>&{VBu}ul9ksx;?@Krq=k_nzgQh0^EiEcu^V_#`IhyoR9gMnFh(%m~V`Kgx{geZ& zMW2n$sKNbbWFmh4Jzx-Vzmb)71obDN{sPpk;dv*$)ce$fRBuAkO}$5Xxp6}UF){PG zZqiQtM~@EhHOaE@<7&n?+pzkt1R} z12WD}`X^3&Z9aC};09xCNdDm9Rl;rCKCe(JcSBazu0Dk#3bM1KXbuWRG~&*FBzrex zX70r2yW#7v=Wt%qPBCB}DF4VWutV`vahR?l`Nxg36ukTHo34?OOKUSSg5c~~zrM7z z07y&o#}NSO=>vm&WD};Od;vRm%FOzSf6^pge>`qXgu_(xA^A-~u3P8r92>jps#+Ze z`T4%Q~yc(u>Vc{Hyrn+o~Jy#);~x1?UkGy zXm+k$J9p-R1E1Cw6@?RLUtyt<$A}?7Nb%4x`Ocr;z+*^EH0w9*M==n;6vt$f^HmrX zwcPApy?TywOw3=dQ65O|tB|_+*RH!@Bz1_6FIAI86*|LWNw zGcxicv%PAS*HnLhdFYlc3nMmfUKnl~X!H4|O}^nFA&bH{ZCVf>9BjlDD?Gvk0%rbA zX5Nem=;PO7!2a*E$jA@*Lu>2N0xvHIiw{1SWwB(5o5j+lgsx^>yx7I!qmSGz7AsGE90KnuHp2TM8|D2Y8{_>m JuEW8Z{{ueS$#MVy literal 0 HcmV?d00001 diff --git a/doc/conf.py b/doc/conf.py index 0fd5eaf05d7..36c0d42b808 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -175,7 +175,7 @@ # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +html_favicon = '_static/favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From ecb10e347bbe0f0e4bab8a358f406923e5468dcf Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 18 May 2018 07:48:10 -0700 Subject: [PATCH 120/282] fix unlimited dims bug (#2154) --- doc/whats-new.rst | 4 +++- xarray/backends/api.py | 3 +++ xarray/tests/test_backends.py | 12 ++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 48abb892350..fe75507e59e 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -39,7 +39,9 @@ Enhancements Bug fixes ~~~~~~~~~ - +- Fixed a bug where `to_netcdf(..., unlimited_dims='bar'` yielded NetCDF files + with spurious 0-length dimensions (i.e. `b`, `a`, and `r`) (:issue:`2134`). + By `Joe Hamman `_. .. _whats-new.0.10.4: diff --git a/xarray/backends/api.py b/xarray/backends/api.py index dec63a85d6e..c3b2aa59fcd 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -686,6 +686,9 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, if unlimited_dims is None: unlimited_dims = dataset.encoding.get('unlimited_dims', None) + if isinstance(unlimited_dims, basestring): + unlimited_dims = [unlimited_dims] + try: dataset.dump_to_store(store, sync=sync, encoding=encoding, unlimited_dims=unlimited_dims, compute=compute) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 95d92cd8b8a..513f5f0834e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1653,11 +1653,23 @@ def test_encoding_unlimited_dims(self): self.assertEqual(actual.encoding['unlimited_dims'], set('y')) assert_equal(ds, actual) + # Regression test for https://github.com/pydata/xarray/issues/2134 + with self.roundtrip(ds, + save_kwargs=dict(unlimited_dims='y')) as actual: + self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert_equal(ds, actual) + ds.encoding = {'unlimited_dims': ['y']} with self.roundtrip(ds) as actual: self.assertEqual(actual.encoding['unlimited_dims'], set('y')) assert_equal(ds, actual) + # Regression test for https://github.com/pydata/xarray/issues/2134 + ds.encoding = {'unlimited_dims': 'y'} + with self.roundtrip(ds) as actual: + self.assertEqual(actual.encoding['unlimited_dims'], set('y')) + assert_equal(ds, actual) + class GenericNetCDFDataTestAutocloseTrue(GenericNetCDFDataTest): autoclose = True From c346d3b7bcdbd6073cf96fdeb0710467a284a611 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 18 May 2018 09:30:42 -1000 Subject: [PATCH 121/282] Bug fixes for Dataset.reduce() and n-dimensional cumsum/cumprod (#2156) * Bug fixes for Dataset.reduce() and n-dimensional cumsum/cumprod Fixes GH1470, "Dataset.mean drops coordinates" Fixes a bug where non-scalar data-variables that did not include the aggregated dimension were not properly reduced: Previously:: >>> ds = Dataset({'x': ('a', [2, 2]), 'y': 2, 'z': ('b', [2])}) >>> ds.var('a') Dimensions: (b: 1) Dimensions without coordinates: b Data variables: x float64 0.0 y float64 0.0 z (b) int64 2 Now:: >>> ds.var('a') Dimensions: (b: 1) Dimensions without coordinates: b Data variables: x int64 0 y int64 0 z (b) int64 0 Finally, adds support for n-dimensional cumsum() and cumprod(), reducing over all dimensions of an array. (This was necessary as part of the above fix.) * Lint fixup * remove confusing comments --- doc/whats-new.rst | 12 ++++++++ xarray/core/dataset.py | 38 +++++++++++------------ xarray/core/duck_array_ops.py | 36 +++++++++++++++++----- xarray/core/variable.py | 5 --- xarray/tests/test_dataarray.py | 5 +++ xarray/tests/test_dataset.py | 34 +++++++++++++++----- xarray/tests/test_duck_array_ops.py | 48 +++++++++++++++++++++++++++++ 7 files changed, 140 insertions(+), 38 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index fe75507e59e..7df47488e21 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -37,12 +37,24 @@ Documentation Enhancements ~~~~~~~~~~~~ +- :py:meth:`~DataArray.cumsum` and :py:meth:`~DataArray.cumprod` now support + aggregation over multiple dimensions at the same time. This is the default + behavior when dimensions are not specified (previously this raised an error). + By `Stephan Hoyer `_ + Bug fixes ~~~~~~~~~ + - Fixed a bug where `to_netcdf(..., unlimited_dims='bar'` yielded NetCDF files with spurious 0-length dimensions (i.e. `b`, `a`, and `r`) (:issue:`2134`). By `Joe Hamman `_. +- Aggregations with :py:meth:`Dataset.reduce` (including ``mean``, ``sum``, + etc) no longer drop unrelated coordinates (:issue:`1470`). Also fixed a + bug where non-scalar data-variables that did not include the aggregation + dimension were improperly skipped. + By `Stephan Hoyer `_ + .. _whats-new.0.10.4: v0.10.4 (May 16, 2018) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index a9ec8c16866..fff11dedb01 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -2594,26 +2594,26 @@ def reduce(self, func, dim=None, keep_attrs=False, numeric_only=False, variables = OrderedDict() for name, var in iteritems(self._variables): reduce_dims = [dim for dim in var.dims if dim in dims] - if reduce_dims or not var.dims: - if name not in self.coords: - if (not numeric_only or - np.issubdtype(var.dtype, np.number) or - (var.dtype == np.bool_)): - if len(reduce_dims) == 1: - # unpack dimensions for the benefit of functions - # like np.argmin which can't handle tuple arguments - reduce_dims, = reduce_dims - elif len(reduce_dims) == var.ndim: - # prefer to aggregate over axis=None rather than - # axis=(0, 1) if they will be equivalent, because - # the former is often more efficient - reduce_dims = None - variables[name] = var.reduce(func, dim=reduce_dims, - keep_attrs=keep_attrs, - allow_lazy=allow_lazy, - **kwargs) + if name in self.coords: + if not reduce_dims: + variables[name] = var else: - variables[name] = var + if (not numeric_only or + np.issubdtype(var.dtype, np.number) or + (var.dtype == np.bool_)): + if len(reduce_dims) == 1: + # unpack dimensions for the benefit of functions + # like np.argmin which can't handle tuple arguments + reduce_dims, = reduce_dims + elif len(reduce_dims) == var.ndim: + # prefer to aggregate over axis=None rather than + # axis=(0, 1) if they will be equivalent, because + # the former is often more efficient + reduce_dims = None + variables[name] = var.reduce(func, dim=reduce_dims, + keep_attrs=keep_attrs, + allow_lazy=allow_lazy, + **kwargs) coord_names = set(k for k in self.coords if k in variables) attrs = self.attrs if keep_attrs else None diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index ef52b4890ef..69b0d0825be 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -281,8 +281,7 @@ def _nanvar_object(value, axis=None, **kwargs): def _create_nan_agg_method(name, numeric_only=False, np_compat=False, - no_bottleneck=False, coerce_strings=False, - keep_dims=False): + no_bottleneck=False, coerce_strings=False): def f(values, axis=None, skipna=None, **kwargs): if kwargs.pop('out', None) is not None: raise TypeError('`out` is not valid for {}'.format(name)) @@ -343,7 +342,6 @@ def f(values, axis=None, skipna=None, **kwargs): 'or newer to use skipna=True or skipna=None' % name) raise NotImplementedError(msg) f.numeric_only = numeric_only - f.keep_dims = keep_dims f.__name__ = name return f @@ -358,10 +356,34 @@ def f(values, axis=None, skipna=None, **kwargs): var = _create_nan_agg_method('var', numeric_only=True) median = _create_nan_agg_method('median', numeric_only=True) prod = _create_nan_agg_method('prod', numeric_only=True, no_bottleneck=True) -cumprod = _create_nan_agg_method('cumprod', numeric_only=True, np_compat=True, - no_bottleneck=True, keep_dims=True) -cumsum = _create_nan_agg_method('cumsum', numeric_only=True, np_compat=True, - no_bottleneck=True, keep_dims=True) +cumprod_1d = _create_nan_agg_method( + 'cumprod', numeric_only=True, np_compat=True, no_bottleneck=True) +cumsum_1d = _create_nan_agg_method( + 'cumsum', numeric_only=True, np_compat=True, no_bottleneck=True) + + +def _nd_cum_func(cum_func, array, axis, **kwargs): + array = asarray(array) + if axis is None: + axis = tuple(range(array.ndim)) + if isinstance(axis, int): + axis = (axis,) + + out = array + for ax in axis: + out = cum_func(out, axis=ax, **kwargs) + return out + + +def cumprod(array, axis=None, **kwargs): + """N-dimensional version of cumprod.""" + return _nd_cum_func(cumprod_1d, array, axis, **kwargs) + + +def cumsum(array, axis=None, **kwargs): + """N-dimensional version of cumsum.""" + return _nd_cum_func(cumsum_1d, array, axis, **kwargs) + _fail_on_dask_array_input_skipna = partial( fail_on_dask_array_input, diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 622ac60d7f6..9dcb99459d4 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1256,11 +1256,6 @@ def reduce(self, func, dim=None, axis=None, keep_attrs=False, if dim is not None and axis is not None: raise ValueError("cannot supply both 'axis' and 'dim' arguments") - if getattr(func, 'keep_dims', False): - if dim is None and axis is None: - raise ValueError("must supply either single 'dim' or 'axis' " - "argument to %s" % (func.__name__)) - if dim is not None: axis = self.get_axis_num(dim) data = func(self.data if allow_lazy else self.values, diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index f2e076db78a..35e270f0db7 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1767,6 +1767,11 @@ def test_cumops(self): orig = DataArray([[-1, 0, 1], [-3, 0, 3]], coords, dims=['x', 'y']) + actual = orig.cumsum() + expected = DataArray([[-1, -1, 0], [-4, -4, 0]], coords, + dims=['x', 'y']) + assert_identical(expected, actual) + actual = orig.cumsum('x') expected = DataArray([[-1, 0, 1], [-4, 0, 4]], coords, dims=['x', 'y']) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 3335a55e4ab..76e41c43c6d 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -3331,7 +3331,18 @@ def test_reduce(self): assert_equal(data.mean(dim=[]), data) - # uint support + def test_reduce_coords(self): + # regression test for GH1470 + data = xr.Dataset({'a': ('x', [1, 2, 3])}, coords={'b': 4}) + expected = xr.Dataset({'a': 2}, coords={'b': 4}) + actual = data.mean('x') + assert_identical(actual, expected) + + # should be consistent + actual = data['a'].mean('x').to_dataset() + assert_identical(actual, expected) + + def test_mean_uint_dtype(self): data = xr.Dataset({'a': (('x', 'y'), np.arange(6).reshape(3, 2).astype('uint')), 'b': (('x', ), np.array([0.1, 0.2, np.nan]))}) @@ -3345,15 +3356,20 @@ def test_reduce_bad_dim(self): with raises_regex(ValueError, 'Dataset does not contain'): data.mean(dim='bad_dim') + def test_reduce_cumsum(self): + data = xr.Dataset({'a': 1, + 'b': ('x', [1, 2]), + 'c': (('x', 'y'), [[np.nan, 3], [0, 4]])}) + assert_identical(data.fillna(0), data.cumsum('y')) + + expected = xr.Dataset({'a': 1, + 'b': ('x', [1, 3]), + 'c': (('x', 'y'), [[0, 3], [0, 7]])}) + assert_identical(expected, data.cumsum()) + def test_reduce_cumsum_test_dims(self): data = create_test_data() for cumfunc in ['cumsum', 'cumprod']: - with raises_regex(ValueError, - "must supply either single 'dim' or 'axis'"): - getattr(data, cumfunc)() - with raises_regex(ValueError, - "must supply either single 'dim' or 'axis'"): - getattr(data, cumfunc)(dim=['dim1', 'dim2']) with raises_regex(ValueError, 'Dataset does not contain'): getattr(data, cumfunc)(dim='bad_dim') @@ -3460,6 +3476,10 @@ def test_reduce_scalars(self): actual = ds.var() assert_identical(expected, actual) + expected = Dataset({'x': 0, 'y': 0, 'z': ('b', [0])}) + actual = ds.var('a') + assert_identical(expected, actual) + def test_reduce_only_one_axis(self): def mean_only_one_axis(x, axis): diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 2983e1991f1..3f4adee6713 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -8,6 +8,7 @@ import warnings from xarray import DataArray, concat +from xarray.core import duck_array_ops from xarray.core.duck_array_ops import ( array_notnull_equiv, concatenate, count, first, last, mean, rolling_window, stack, where) @@ -103,6 +104,53 @@ def test_all_nan_arrays(self): assert np.isnan(mean([np.nan, np.nan])) +def test_cumsum_1d(): + inputs = np.array([0, 1, 2, 3]) + expected = np.array([0, 1, 3, 6]) + actual = duck_array_ops.cumsum(inputs) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumsum(inputs, axis=0) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumsum(inputs, axis=-1) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumsum(inputs, axis=(0,)) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumsum(inputs, axis=()) + assert_array_equal(inputs, actual) + + +def test_cumsum_2d(): + inputs = np.array([[1, 2], [3, 4]]) + + expected = np.array([[1, 3], [4, 10]]) + actual = duck_array_ops.cumsum(inputs) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumsum(inputs, axis=(0, 1)) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumsum(inputs, axis=()) + assert_array_equal(inputs, actual) + + +def test_cumprod_2d(): + inputs = np.array([[1, 2], [3, 4]]) + + expected = np.array([[1, 2], [3, 2 * 3 * 4]]) + actual = duck_array_ops.cumprod(inputs) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumprod(inputs, axis=(0, 1)) + assert_array_equal(expected, actual) + + actual = duck_array_ops.cumprod(inputs, axis=()) + assert_array_equal(inputs, actual) + + class TestArrayNotNullEquiv(): @pytest.mark.parametrize("arr1, arr2", [ (np.array([1, 2, 3]), np.array([1, 2, 3])), From 585b9a7913d98e26c28b4f1da599c1c6db551362 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 20 May 2018 16:14:02 -0700 Subject: [PATCH 122/282] Versioneer (#2163) * add versioneer to simplify/standardize package versioning * reorg __init__.py for version import * fix for docs * what is new --- .gitattributes | 1 + HOW_TO_RELEASE | 28 +- MANIFEST.in | 2 + doc/conf.py | 2 +- doc/whats-new.rst | 4 + setup.cfg | 11 + setup.py | 87 +-- versioneer.py | 1822 ++++++++++++++++++++++++++++++++++++++++++++ xarray/__init__.py | 11 +- xarray/_version.py | 520 +++++++++++++ 10 files changed, 2380 insertions(+), 108 deletions(-) create mode 100644 versioneer.py create mode 100644 xarray/_version.py diff --git a/.gitattributes b/.gitattributes index a52f4ca283a..daa5b82874e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # reduce the number of merge conflicts doc/whats-new.rst merge=union +xarray/_version.py export-subst diff --git a/HOW_TO_RELEASE b/HOW_TO_RELEASE index f1fee59e177..cdfcace809a 100644 --- a/HOW_TO_RELEASE +++ b/HOW_TO_RELEASE @@ -7,21 +7,20 @@ Time required: about an hour. 2. Look over whats-new.rst and the docs. Make sure "What's New" is complete (check the date!) and add a brief summary note describing the release at the top. - 3. Update the version in setup.py and switch to `ISRELEASED = True`. - 4. If you have any doubts, run the full test suite one final time! + 3. If you have any doubts, run the full test suite one final time! py.test - 5. On the master branch, commit the release in git: + 4. On the master branch, commit the release in git: git commit -a -m 'Release v0.X.Y' - 6. Tag the release: + 5. Tag the release: git tag -a v0.X.Y -m 'v0.X.Y' - 7. Build source and binary wheels for pypi: + 6. Build source and binary wheels for pypi: python setup.py bdist_wheel sdist - 8. Use twine to register and upload the release on pypi. Be careful, you can't + 7. Use twine to register and upload the release on pypi. Be careful, you can't take this back! twine upload dist/xarray-0.X.Y* You will need to be listed as a package owner at https://pypi.python.org/pypi/xarray for this to work. - 9. Push your changes to master: + 8. Push your changes to master: git push upstream master git push upstream --tags 9. Update the stable branch (used by ReadTheDocs) and switch back to master: @@ -32,25 +31,22 @@ Time required: about an hour. It's OK to force push to 'stable' if necessary. We also update the stable branch with `git cherrypick` for documentation only fixes that apply the current released version. -10. Revert ISRELEASED in setup.py back to False. Don't change the version - number: in normal development, we keep the version number in setup.py as the - last released version. -11. Add a section for the next release (v.X.(Y+1)) to doc/whats-new.rst. -12. Commit your changes and push to master again: +10. Add a section for the next release (v.X.(Y+1)) to doc/whats-new.rst. +11. Commit your changes and push to master again: git commit -a -m 'Revert to dev version' git push upstream master You're done pushing to master! -13. Issue the release on GitHub. Click on "Draft a new release" at +12. Issue the release on GitHub. Click on "Draft a new release" at https://github.com/pydata/xarray/releases and paste in the latest from whats-new.rst. -14. Update the docs. Login to https://readthedocs.org/projects/xray/versions/ +13. Update the docs. Login to https://readthedocs.org/projects/xray/versions/ and switch your new release tag (at the bottom) from "Inactive" to "Active". It should now build automatically. -15. Update conda-forge. Clone https://github.com/conda-forge/xarray-feedstock +14. Update conda-forge. Clone https://github.com/conda-forge/xarray-feedstock and update the version number and sha256 in meta.yaml. (On OS X, you can calculate sha256 with `shasum -a 256 xarray-0.X.Y.tar.gz`). Submit a pull request (and merge it, once CI passes). -16. Issue the release announcement! For bug fix releases, I usually only email +15. Issue the release announcement! For bug fix releases, I usually only email xarray@googlegroups.com. For major/feature releases, I will email a broader list (no more than once every 3-6 months): pydata@googlegroups.com, xarray@googlegroups.com, diff --git a/MANIFEST.in b/MANIFEST.in index a49c49cd396..a006660e5fb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,5 @@ recursive-include doc * prune doc/_build prune doc/generated global-exclude .DS_Store +include versioneer.py +include xarray/_version.py diff --git a/doc/conf.py b/doc/conf.py index 36c0d42b808..5fd3bece3bd 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -102,7 +102,7 @@ # built documents. # # The short X.Y version. -version = xarray.version.short_version +version = xarray.__version__.split('+')[0] # The full version, including alpha/beta/rc tags. release = xarray.__version__ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 7df47488e21..4c9a1415e26 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -42,6 +42,10 @@ Enhancements behavior when dimensions are not specified (previously this raised an error). By `Stephan Hoyer `_ +- Xarray now uses `Versioneer `__ + to manage its version strings. (:issue:`1300`). + By `Joe Hamman `_. + Bug fixes ~~~~~~~~~ diff --git a/setup.cfg b/setup.cfg index ec30a10b242..850551b3579 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,3 +15,14 @@ exclude= default_section=THIRDPARTY known_first_party=xarray multi_line_output=4 + +[versioneer] +VCS = git +style = pep440 +versionfile_source = xarray/_version.py +versionfile_build = xarray/_version.py +tag_prefix = +parentdir_prefix = xarray- + +[aliases] +test = pytest diff --git a/setup.py b/setup.py index b5130958c00..77c6083f52c 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,9 @@ #!/usr/bin/env python -import os -import re import sys -import warnings from setuptools import find_packages, setup -MAJOR = 0 -MINOR = 10 -MICRO = 4 -ISRELEASED = False -VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) -QUALIFIER = '' +import versioneer DISTNAME = 'xarray' @@ -65,83 +57,10 @@ - SciPy2015 talk: https://www.youtube.com/watch?v=X0pAhJgySxk """ # noqa -# Code to extract and write the version copied from pandas. -# Used under the terms of pandas's license, see licenses/PANDAS_LICENSE. -FULLVERSION = VERSION -write_version = True - -if not ISRELEASED: - import subprocess - FULLVERSION += '.dev' - - pipe = None - for cmd in ['git', 'git.cmd']: - try: - pipe = subprocess.Popen( - [cmd, "describe", "--always", "--match", "v[0-9]*"], - stdout=subprocess.PIPE) - (so, serr) = pipe.communicate() - if pipe.returncode == 0: - break - except BaseException: - pass - - if pipe is None or pipe.returncode != 0: - # no git, or not in git dir - if os.path.exists('xarray/version.py'): - warnings.warn( - "WARNING: Couldn't get git revision," - " using existing xarray/version.py") - write_version = False - else: - warnings.warn( - "WARNING: Couldn't get git revision," - " using generic version string") - else: - # have git, in git dir, but may have used a shallow clone (travis does - # this) - rev = so.strip() - # makes distutils blow up on Python 2.7 - if sys.version_info[0] >= 3: - rev = rev.decode('ascii') - - if not rev.startswith('v') and re.match("[a-zA-Z0-9]{7,9}", rev): - # partial clone, manually construct version string - # this is the format before we started using git-describe - # to get an ordering on dev version strings. - rev = "v%s+dev.%s" % (VERSION, rev) - - # Strip leading v from tags format "vx.y.z" to get th version string - FULLVERSION = rev.lstrip('v') - - # make sure we respect PEP 440 - FULLVERSION = FULLVERSION.replace("-", "+dev", 1).replace("-", ".") - -else: - FULLVERSION += QUALIFIER - - -def write_version_py(filename=None): - cnt = """\ -version = '%s' -short_version = '%s' -""" - if not filename: - filename = os.path.join( - os.path.dirname(__file__), 'xarray', 'version.py') - - a = open(filename, 'w') - try: - a.write(cnt % (FULLVERSION, VERSION)) - finally: - a.close() - - -if write_version: - write_version_py() setup(name=DISTNAME, - version=FULLVERSION, + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), license=LICENSE, author=AUTHOR, author_email=AUTHOR_EMAIL, diff --git a/versioneer.py b/versioneer.py new file mode 100644 index 00000000000..64fea1c8927 --- /dev/null +++ b/versioneer.py @@ -0,0 +1,1822 @@ + +# Version: 0.18 + +"""The Versioneer - like a rocketeer, but for versions. + +The Versioneer +============== + +* like a rocketeer, but for versions! +* https://github.com/warner/python-versioneer +* Brian Warner +* License: Public Domain +* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere to your $PATH +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes. + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +See [INSTALL.md](./INSTALL.md) for detailed installation instructions. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the + commit date in ISO 8601 format. This will be None if the date is not + available. + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See [details.md](details.md) in the Versioneer +source tree for descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Known Limitations + +Some situations are known to cause problems for Versioneer. This details the +most significant ones. More can be found on Github +[issues page](https://github.com/warner/python-versioneer/issues). + +### Subprojects + +Versioneer has limited support for source trees in which `setup.py` is not in +the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are +two common reasons why `setup.py` might not be in the root: + +* Source trees which contain multiple subprojects, such as + [Buildbot](https://github.com/buildbot/buildbot), which contains both + "master" and "slave" subprojects, each with their own `setup.py`, + `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI + distributions (and upload multiple independently-installable tarballs). +* Source trees whose main purpose is to contain a C library, but which also + provide bindings to Python (and perhaps other langauges) in subdirectories. + +Versioneer will look for `.git` in parent directories, and most operations +should get the right version string. However `pip` and `setuptools` have bugs +and implementation details which frequently cause `pip install .` from a +subproject directory to fail to find a correct version string (so it usually +defaults to `0+unknown`). + +`pip install --editable .` should work correctly. `setup.py install` might +work too. + +Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in +some later version. + +[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +this issue. The discussion in +[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +issue from the Versioneer side in more detail. +[pip PR#3176](https://github.com/pypa/pip/pull/3176) and +[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve +pip to let Versioneer work correctly. + +Versioneer-0.16 and earlier only looked for a `.git` directory next to the +`setup.cfg`, so subprojects were completely unsupported with those releases. + +### Editable installs with setuptools <= 18.5 + +`setup.py develop` and `pip install --editable .` allow you to install a +project into a virtualenv once, then continue editing the source code (and +test) without re-installing after every change. + +"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a +convenient way to specify executable scripts that should be installed along +with the python package. + +These both work as expected when using modern setuptools. When using +setuptools-18.5 or earlier, however, certain operations will cause +`pkg_resources.DistributionNotFound` errors when running the entrypoint +script, which must be resolved by re-installing the package. This happens +when the install happens with one version, then the egg_info data is +regenerated while a different version is checked out. Many setup.py commands +cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into +a different virtualenv), so this can be surprising. + +[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +this one, but upgrading to a newer version of setuptools should probably +resolve it. + +### Unicode version strings + +While Versioneer works (and is continually tested) with both Python 2 and +Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. +Newer releases probably generate unicode version strings on py2. It's not +clear that this is wrong, but it may be surprising for applications when then +write these strings to a network connection or include them in bytes-oriented +APIs like cryptographic checksums. + +[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates +this question. + + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + + +## License + +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . + +""" + +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(me)[0]) + vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) + if me_dir != vsr_dir: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +LONG_VERSION_PY['git'] = ''' +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.18 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %%s" %% (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %%s (error)" %% dispcmd) + print("stdout was %%s" %% stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %%s but none started with prefix %%s" %% + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %%d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%%s', no digits" %% ",".join(refs - tags)) + if verbose: + print("likely tags: %%s" %% ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %%s" %% r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %%s not under git control" %% root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-subst keyword substitution. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) + present = False + try: + f = open(".gitattributes", "r") + for line in f.readlines(): + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + f.close() + except EnvironmentError: + pass + if not present: + f = open(".gitattributes", "a+") + f.write("%s export-subst\n" % versionfile_source) + f.close() + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +SHORT_VERSION_PY = """ +# This file was generated by 'versioneer.py' (0.18) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +import json + +version_json = ''' +%s +''' # END VERSION_JSON + + +def get_versions(): + return json.loads(version_json) +""" + + +def versions_from_file(filename): + """Try to determine the version from _version.py if present.""" + try: + with open(filename) as f: + contents = f.read() + except EnvironmentError: + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + + +def write_to_version_file(filename, versions): + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + + print("set %s to '%s'" % (filename, versions["version"])) + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: + pass + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: + pass + + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version", + "date": None} + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(): + """Get the custom setuptools/distutils subclasses used by Versioneer.""" + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + print(" date: %s" % vers.get("date")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + # pip install: + # copies source tree to a tempdir before running egg_info/etc + # if .git isn't copied too, 'git describe' will fail + # then does setup.py bdist_wheel, or sometimes setup.py install + # setup.py egg_info -> ? + + # we override different "build_py" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string + # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. + # setup(console=[{ + # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION + # "product_version": versioneer.get_version(), + # ... + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + if 'py2exe' in sys.modules: # py2exe enabled? + try: + from py2exe.distutils_buildexe import py2exe as _py2exe # py3 + except ImportError: + from py2exe.build_exe import py2exe as _py2exe # py2 + + class cmd_py2exe(_py2exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _py2exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["py2exe"] = cmd_py2exe + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" + +INIT_PY_SNIPPET = """ +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions +""" + + +def do_setup(): + """Main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): + try: + with open(ipy, "r") as f: + old = f.read() + except EnvironmentError: + old = "" + if INIT_PY_SNIPPET not in old: + print(" appending to %s" % ipy) + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) + else: + print(" %s unmodified" % ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") + + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-subst keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1) diff --git a/xarray/__init__.py b/xarray/__init__.py index 94e8029edbb..7cc7811b783 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -3,6 +3,10 @@ from __future__ import division from __future__ import print_function +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions + from .core.alignment import align, broadcast, broadcast_arrays from .core.common import full_like, zeros_like, ones_like from .core.combine import concat, auto_combine @@ -24,13 +28,6 @@ from .coding.cftimeindex import CFTimeIndex -try: - from .version import version as __version__ -except ImportError: # pragma: no cover - raise ImportError('xarray not properly installed. If you are running from ' - 'the source directory, please instead create a new ' - 'virtual environment (using conda or virtualenv) and ' - 'then install it in-place by running: pip install -e .') from .util.print_versions import show_versions from . import tutorial diff --git a/xarray/_version.py b/xarray/_version.py new file mode 100644 index 00000000000..2fa32b69798 --- /dev/null +++ b/xarray/_version.py @@ -0,0 +1,520 @@ + +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.18 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "xarray-" + cfg.versionfile_source = "xarray/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} From 48d55eea052fec204b843babdc81c258f3ed5ce1 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 21 May 2018 04:02:34 -0400 Subject: [PATCH 123/282] Fix string slice indexing for a length-1 CFTimeIndex (#2166) * Fix string slice indexing for length-1 CFTimeIndex * Skip test if cftime is not installed * Add a what's new entry --- doc/whats-new.rst | 6 ++++++ xarray/coding/cftimeindex.py | 2 +- xarray/tests/test_cftimeindex.py | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4c9a1415e26..d9f43fa1868 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -59,6 +59,12 @@ Bug fixes dimension were improperly skipped. By `Stephan Hoyer `_ +- Selecting data indexed by a length-1 ``CFTimeIndex`` with a slice of strings + now behaves as it does when using a length-1 ``DatetimeIndex`` (i.e. it no + longer falsely returns an empty array when the slice includes the value in + the index) (:issue:`2165`). + By `Spencer Clark `_. + .. _whats-new.0.10.4: v0.10.4 (May 16, 2018) diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 5fca14ddbb1..eb8cae2f398 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -225,7 +225,7 @@ def _maybe_cast_slice_bound(self, label, side, kind): label) start, end = _parsed_string_to_bounds(self.date_type, resolution, parsed) - if self.is_monotonic_decreasing and len(self): + if self.is_monotonic_decreasing and len(self) > 1: return end if side == 'left' else start return start if side == 'left' else end else: diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index c78ac038bd5..6f102b60b9d 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -79,6 +79,12 @@ def monotonic_decreasing_index(date_type): return CFTimeIndex(dates) +@pytest.fixture +def length_one_index(date_type): + dates = [date_type(1, 1, 1)] + return CFTimeIndex(dates) + + @pytest.fixture def da(index): return xr.DataArray([1, 2, 3, 4], coords=[index], @@ -280,6 +286,36 @@ def test_get_slice_bound_decreasing_index( assert result == expected +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('kind', ['loc', 'getitem']) +def test_get_slice_bound_length_one_index( + date_type, length_one_index, kind): + result = length_one_index.get_slice_bound('0001', 'left', kind) + expected = 0 + assert result == expected + + result = length_one_index.get_slice_bound('0001', 'right', kind) + expected = 1 + assert result == expected + + result = length_one_index.get_slice_bound( + date_type(1, 3, 1), 'left', kind) + expected = 1 + assert result == expected + + result = length_one_index.get_slice_bound( + date_type(1, 3, 1), 'right', kind) + expected = 1 + assert result == expected + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +def test_string_slice_length_one_index(length_one_index): + da = xr.DataArray([1], coords=[length_one_index], dims=['time']) + result = da.sel(time=slice('0001', '0001')) + assert_identical(result, da) + + @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_date_type_property(date_type, index): assert index.date_type is date_type From b48e0969670f17857a314b5a755b1a1bf7ee38df Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 24 May 2018 17:52:06 -0700 Subject: [PATCH 124/282] BUG: fix writing to groups with h5netcdf (#2181) * BUG: fix writing to groups with h5netcdf Fixes GH2177 Our test suite was inadvertently not checking this. * what's new note --- doc/whats-new.rst | 6 +++++- xarray/backends/h5netcdf_.py | 9 +++++++-- xarray/backends/netCDF4_.py | 10 +++++++--- xarray/tests/test_backends.py | 12 ++++++------ 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d9f43fa1868..4a01065bd70 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -64,7 +64,11 @@ Bug fixes longer falsely returns an empty array when the slice includes the value in the index) (:issue:`2165`). By `Spencer Clark `_. - + +- Fix Dataset.to_netcdf() cannot create group with engine="h5netcdf" + (:issue:`2177`). + By `Stephan Hoyer `_ + .. _whats-new.0.10.4: v0.10.4 (May 16, 2018) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index f9e2b3dece1..6b3cd9ebb15 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -12,7 +12,7 @@ HDF5_LOCK, DataStorePickleMixin, WritableCFDataStore, find_root) from .netCDF4_ import ( BaseNetCDF4Array, _encode_nc4_variable, _extract_nc4_variable_encoding, - _get_datatype, _nc4_group) + _get_datatype, _nc4_require_group) class H5NetCDFArrayWrapper(BaseNetCDF4Array): @@ -57,11 +57,16 @@ def _read_attributes(h5netcdf_var): lsd_okay=False, h5py_okay=True, backend='h5netcdf') +def _h5netcdf_create_group(dataset, name): + return dataset.create_group(name) + + def _open_h5netcdf_group(filename, mode, group): import h5netcdf ds = h5netcdf.File(filename, mode=mode) with close_on_error(ds): - return _nc4_group(ds, group, mode) + return _nc4_require_group( + ds, group, mode, create_group=_h5netcdf_create_group) class H5NetCDFStore(WritableCFDataStore, DataStorePickleMixin): diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 14061a0fb08..5391a890fb3 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -108,7 +108,11 @@ def _nc4_dtype(var): return dtype -def _nc4_group(ds, group, mode): +def _netcdf4_create_group(dataset, name): + return dataset.createGroup(name) + + +def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group): if group in set([None, '', '/']): # use the root group return ds @@ -123,7 +127,7 @@ def _nc4_group(ds, group, mode): ds = ds.groups[key] except KeyError as e: if mode != 'r': - ds = ds.createGroup(key) + ds = create_group(ds, key) else: # wrap error to provide slightly more helpful message raise IOError('group not found: %s' % key, e) @@ -210,7 +214,7 @@ def _open_netcdf4_group(filename, mode, group=None, **kwargs): ds = nc4.Dataset(filename, mode=mode, **kwargs) with close_on_error(ds): - ds = _nc4_group(ds, group, mode) + ds = _nc4_require_group(ds, group, mode) _disable_auto_decode_group(ds) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 513f5f0834e..0768a942a77 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -892,7 +892,7 @@ def test_open_group(self): # check equivalent ways to specify group for group in 'foo', '/foo', 'foo/', '/foo/': - with open_dataset(tmp_file, group=group) as actual: + with self.open(tmp_file, group=group) as actual: assert_equal(actual['x'], expected['x']) # check that missing group raises appropriate exception @@ -920,18 +920,18 @@ def test_open_subgroup(self): # check equivalent ways to specify group for group in 'foo/bar', '/foo/bar', 'foo/bar/', '/foo/bar/': - with open_dataset(tmp_file, group=group) as actual: + with self.open(tmp_file, group=group) as actual: assert_equal(actual['x'], expected['x']) def test_write_groups(self): data1 = create_test_data() data2 = data1 * 2 with create_tmp_file() as tmp_file: - data1.to_netcdf(tmp_file, group='data/1') - data2.to_netcdf(tmp_file, group='data/2', mode='a') - with open_dataset(tmp_file, group='data/1') as actual1: + self.save(data1, tmp_file, group='data/1') + self.save(data2, tmp_file, group='data/2', mode='a') + with self.open(tmp_file, group='data/1') as actual1: assert_identical(data1, actual1) - with open_dataset(tmp_file, group='data/2') as actual2: + with self.open(tmp_file, group='data/2') as actual2: assert_identical(data2, actual2) def test_roundtrip_string_with_fill_value_vlen(self): From 04df50efefecaea729133c14082eb5e24491633e Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 25 May 2018 19:38:48 +0900 Subject: [PATCH 125/282] weighted rolling mean -> weighted rolling sum (#2185) An example of weighted rolling mean in doc is actually weighted rolling *sum*. It is a little bit misleading (SO)[https://stackoverflow.com/questions/50520835/xarray-simple-weighted-rolling-mean-example-using-construct/50524093#50524093], so I propose to change `weighted rolling mean` -> `weighted rolling sum` --- doc/computation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/computation.rst b/doc/computation.rst index 0f22a2ed967..6793e667e06 100644 --- a/doc/computation.rst +++ b/doc/computation.rst @@ -185,7 +185,7 @@ windowed rolling, convolution, short-time FFT etc. Because the ``DataArray`` given by ``r.construct('window_dim')`` is a view of the original array, it is memory efficient. -You can also use ``construct`` to compute a weighted rolling mean: +You can also use ``construct`` to compute a weighted rolling sum: .. ipython:: python From a28aab005b42eabe0b1651d2330ed2f3268bb9f8 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 25 May 2018 20:29:45 -0700 Subject: [PATCH 126/282] Fix DataArray.stack() with non-unique coordinates on pandas 0.23 (#2168) --- doc/whats-new.rst | 4 ++++ xarray/core/utils.py | 14 ++++++++------ xarray/tests/test_dataarray.py | 7 +++++++ xarray/tests/test_utils.py | 12 +++++++++++- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4a01065bd70..055369f0352 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -59,6 +59,10 @@ Bug fixes dimension were improperly skipped. By `Stephan Hoyer `_ +- Fix :meth:`~DataArray.stack` with non-unique coordinates on pandas 0.23 + (:issue:`2160`). + By `Stephan Hoyer `_ + - Selecting data indexed by a length-1 ``CFTimeIndex`` with a slice of strings now behaves as it does when using a length-1 ``DatetimeIndex`` (i.e. it no longer falsely returns an empty array when the slice includes the value in diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 06bb3ede393..f6c5830cc9e 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -76,13 +76,12 @@ def safe_cast_to_index(array): def multiindex_from_product_levels(levels, names=None): """Creating a MultiIndex from a product without refactorizing levels. - Keeping levels the same is faster, and also gives back the original labels - when we unstack. + Keeping levels the same gives back the original labels when we unstack. Parameters ---------- - levels : sequence of arrays - Unique labels for each level. + levels : sequence of pd.Index + Values for each MultiIndex level. names : optional sequence of objects Names for each level. @@ -90,8 +89,11 @@ def multiindex_from_product_levels(levels, names=None): ------- pandas.MultiIndex """ - labels_mesh = np.meshgrid(*[np.arange(len(lev)) for lev in levels], - indexing='ij') + if any(not isinstance(lev, pd.Index) for lev in levels): + raise TypeError('levels must be a list of pd.Index objects') + + split_labels, levels = zip(*[lev.factorize() for lev in levels]) + labels_mesh = np.meshgrid(*split_labels, indexing='ij') labels = [x.ravel() for x in labels_mesh] return pd.MultiIndex(levels, labels, sortorder=0, names=names) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 35e270f0db7..a03d265c3e3 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1673,6 +1673,13 @@ def test_unstack_pandas_consistency(self): actual = DataArray(s, dims='z').unstack('z') assert_identical(expected, actual) + def test_stack_nonunique_consistency(self): + orig = DataArray([[0, 1], [2, 3]], dims=['x', 'y'], + coords={'x': [0, 1], 'y': [0, 0]}) + actual = orig.stack(z=['x', 'y']) + expected = DataArray(orig.to_pandas().stack(), dims='z') + assert_identical(expected, actual) + def test_transpose(self): assert_equal(self.dv.variable.transpose(), self.dv.transpose().variable) diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 0b3b0ee7dd6..1f73743d01d 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -72,7 +72,8 @@ def test_safe_cast_to_index_datetime_datetime(enable_cftimeindex): def test_multiindex_from_product_levels(): - result = utils.multiindex_from_product_levels([['b', 'a'], [1, 3, 2]]) + result = utils.multiindex_from_product_levels( + [pd.Index(['b', 'a']), pd.Index([1, 3, 2])]) np.testing.assert_array_equal( result.labels, [[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]]) np.testing.assert_array_equal(result.levels[0], ['b', 'a']) @@ -82,6 +83,15 @@ def test_multiindex_from_product_levels(): np.testing.assert_array_equal(result.values, other.values) +def test_multiindex_from_product_levels_non_unique(): + result = utils.multiindex_from_product_levels( + [pd.Index(['b', 'a']), pd.Index([1, 1, 2])]) + np.testing.assert_array_equal( + result.labels, [[0, 0, 0, 1, 1, 1], [0, 0, 1, 0, 0, 1]]) + np.testing.assert_array_equal(result.levels[0], ['b', 'a']) + np.testing.assert_array_equal(result.levels[1], [1, 2]) + + class TestArrayEquiv(TestCase): def test_0d(self): # verify our work around for pd.isnull not working for 0-dimensional From a8c1ed2ae3cc15863d37d869a7e1658eb33e01f6 Mon Sep 17 00:00:00 2001 From: Johnnie Gray Date: Sun, 27 May 2018 21:45:37 +0100 Subject: [PATCH 127/282] add xyzpy to projects (#2189) --- doc/faq.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/faq.rst b/doc/faq.rst index 9d763f1c15f..170a1e17bdc 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -211,6 +211,7 @@ Extend xarray capabilities - `xrft `_: Fourier transforms for xarray data. - `xr-scipy `_: A lightweight scipy wrapper for xarray. - `X-regression `_: Multiple linear regression from Statsmodels library coupled with Xarray library. +- `xyzpy `_: Easily generate high dimensional data, including parallelization. Visualization ~~~~~~~~~~~~~ From 847050026d45e2817960a37564bd8e909ecbdb05 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Sun, 27 May 2018 16:48:30 -0400 Subject: [PATCH 128/282] Datasets more robust to non-string keys (#2174) * ds more robust to non-str keys * formatting * time.dayofyear needs cover in dataarray getitem * trial of indexer_dict * feedback from stephan * a few more methods * reindex added * rename to either_dict_or_kwargs * remove assert check * docstring * more docstring * `optional` goes last * last docstring * what's new * artefact * test either_dict_or_kwargs --- doc/whats-new.rst | 8 +++++ setup.cfg | 1 + xarray/core/coordinates.py | 8 +++-- xarray/core/dataarray.py | 42 +++++++++++++++-------- xarray/core/dataset.py | 63 +++++++++++++++++++++------------- xarray/core/utils.py | 2 +- xarray/core/variable.py | 8 +++-- xarray/tests/test_dataarray.py | 4 +-- xarray/tests/test_dataset.py | 8 ++++- xarray/tests/test_utils.py | 24 +++++++++++-- 10 files changed, 116 insertions(+), 52 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 055369f0352..68bf5318bf5 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -46,6 +46,14 @@ Enhancements to manage its version strings. (:issue:`1300`). By `Joe Hamman `_. +- :py:meth:`~DataArray.sel`, :py:meth:`~DataArray.isel` & :py:meth:`~DataArray.reindex`, + (and their :py:class:`Dataset` counterparts) now support supplying a ``dict`` + as a first argument, as an alternative to the existing approach + of supplying a set of `kwargs`. This allows for more robust behavior + of dimension names which conflict with other keyword names, or are + not strings. + By `Maximilian Roos `_. + Bug fixes ~~~~~~~~~ diff --git a/setup.cfg b/setup.cfg index 850551b3579..4dd1bffe043 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,6 +8,7 @@ testpaths=xarray/tests [flake8] max-line-length=79 ignore= + W503 exclude= doc/ diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index cb22c0b687b..efe8affb2a3 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -9,10 +9,9 @@ from .merge import ( expand_and_merge_variables, merge_coords, merge_coords_for_inplace_math) from .pycompat import OrderedDict -from .utils import Frozen, ReprObject +from .utils import Frozen, ReprObject, either_dict_or_kwargs from .variable import Variable - # Used as the key corresponding to a DataArray's variable when converting # arbitrary DataArray objects to datasets _THIS_ARRAY = ReprObject('') @@ -332,7 +331,8 @@ def assert_coordinate_consistent(obj, coords): .format(k, obj[k], coords[k])) -def remap_label_indexers(obj, method=None, tolerance=None, **indexers): +def remap_label_indexers(obj, indexers=None, method=None, tolerance=None, + **indexers_kwargs): """ Remap **indexers from obj.coords. If indexer is an instance of DataArray and it has coordinate, then this @@ -345,6 +345,8 @@ def remap_label_indexers(obj, method=None, tolerance=None, **indexers): new_indexes: mapping of new dimensional-coordinate. """ from .dataarray import DataArray + indexers = either_dict_or_kwargs( + indexers, indexers_kwargs, 'remap_label_indexers') v_indexers = {k: v.variable.data if isinstance(v, DataArray) else v for k, v in indexers.items()} diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index fc7091dad85..da9acb48a7a 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -18,7 +18,9 @@ from .formatting import format_item from .options import OPTIONS from .pycompat import OrderedDict, basestring, iteritems, range, zip -from .utils import decode_numpy_dict_values, ensure_us_time_resolution +from .utils import ( + either_dict_or_kwargs, decode_numpy_dict_values, + ensure_us_time_resolution) from .variable import ( IndexVariable, Variable, as_compatible_data, as_variable, assert_unique_multiindex_level_names) @@ -470,7 +472,7 @@ def __getitem__(self, key): return self._getitem_coord(key) else: # xarray-style array indexing - return self.isel(**self._item_key_to_dict(key)) + return self.isel(indexers=self._item_key_to_dict(key)) def __setitem__(self, key, value): if isinstance(key, basestring): @@ -498,7 +500,7 @@ def _attr_sources(self): @property def _item_sources(self): """List of places to look-up items for key-completion""" - return [self.coords, {d: self[d] for d in self.dims}, + return [self.coords, {d: self.coords[d] for d in self.dims}, LevelCoordinatesSource(self)] def __contains__(self, key): @@ -742,7 +744,7 @@ def chunk(self, chunks=None, name_prefix='xarray-', token=None, token=token, lock=lock) return self._from_temp_dataset(ds) - def isel(self, drop=False, **indexers): + def isel(self, indexers=None, drop=False, **indexers_kwargs): """Return a new DataArray whose dataset is given by integer indexing along the specified dimension(s). @@ -751,10 +753,12 @@ def isel(self, drop=False, **indexers): Dataset.isel DataArray.sel """ - ds = self._to_temp_dataset().isel(drop=drop, **indexers) + indexers = either_dict_or_kwargs(indexers, indexers_kwargs, 'isel') + ds = self._to_temp_dataset().isel(drop=drop, indexers=indexers) return self._from_temp_dataset(ds) - def sel(self, method=None, tolerance=None, drop=False, **indexers): + def sel(self, indexers=None, method=None, tolerance=None, drop=False, + **indexers_kwargs): """Return a new DataArray whose dataset is given by selecting index labels along the specified dimension(s). @@ -776,8 +780,9 @@ def sel(self, method=None, tolerance=None, drop=False, **indexers): DataArray.isel """ - ds = self._to_temp_dataset().sel(drop=drop, method=method, - tolerance=tolerance, **indexers) + indexers = either_dict_or_kwargs(indexers, indexers_kwargs, 'sel') + ds = self._to_temp_dataset().sel( + indexers=indexers, drop=drop, method=method, tolerance=tolerance) return self._from_temp_dataset(ds) def isel_points(self, dim='points', **indexers): @@ -851,12 +856,19 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): return self.reindex(method=method, tolerance=tolerance, copy=copy, **indexers) - def reindex(self, method=None, tolerance=None, copy=True, **indexers): + def reindex(self, indexers=None, method=None, tolerance=None, copy=True, + **indexers_kwargs): """Conform this object onto a new set of indexes, filling in missing values with NaN. Parameters ---------- + indexers : dict, optional + Dictionary with keys given by dimension names and values given by + arrays of coordinates tick labels. Any mis-matched coordinate + values will be filled in with NaN, and any mis-matched dimension + names will simply be ignored. + One of indexers or indexers_kwargs must be provided. copy : bool, optional If ``copy=True``, data in the return value is always copied. If ``copy=False`` and reindexing is unnecessary, or can be performed @@ -874,11 +886,9 @@ def reindex(self, method=None, tolerance=None, copy=True, **indexers): Maximum distance between original and new labels for inexact matches. The values of the index at the matching locations most satisfy the equation ``abs(index[indexer] - target) <= tolerance``. - **indexers : dict - Dictionary with keys given by dimension names and values given by - arrays of coordinates tick labels. Any mis-matched coordinate - values will be filled in with NaN, and any mis-matched dimension - names will simply be ignored. + **indexers_kwarg : {dim: indexer, ...}, optional + The keyword arguments form of ``indexers``. + One of indexers or indexers_kwargs must be provided. Returns ------- @@ -891,8 +901,10 @@ def reindex(self, method=None, tolerance=None, copy=True, **indexers): DataArray.reindex_like align """ + indexers = either_dict_or_kwargs( + indexers, indexers_kwargs, 'reindex') ds = self._to_temp_dataset().reindex( - method=method, tolerance=tolerance, copy=copy, **indexers) + indexers=indexers, method=method, tolerance=tolerance, copy=copy) return self._from_temp_dataset(ds) def rename(self, new_name_or_name_dict): diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index fff11dedb01..d6a5ac1c172 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -17,8 +17,8 @@ rolling, utils) from .. import conventions from .alignment import align -from .common import (DataWithCoords, ImplementsDatasetReduce, - _contains_datetime_like_objects) +from .common import ( + DataWithCoords, ImplementsDatasetReduce, _contains_datetime_like_objects) from .coordinates import ( DatasetCoordinates, Indexes, LevelCoordinatesSource, assert_coordinate_consistent, remap_label_indexers) @@ -30,7 +30,7 @@ from .pycompat import ( OrderedDict, basestring, dask_array_type, integer_types, iteritems, range) from .utils import ( - Frozen, SortedKeysDict, decode_numpy_dict_values, + Frozen, SortedKeysDict, either_dict_or_kwargs, decode_numpy_dict_values, ensure_us_time_resolution, hashable, maybe_wrap_array) from .variable import IndexVariable, Variable, as_variable, broadcast_variables @@ -1368,7 +1368,7 @@ def _get_indexers_coordinates(self, indexers): attached_coords[k] = v return attached_coords - def isel(self, drop=False, **indexers): + def isel(self, indexers=None, drop=False, **indexers_kwargs): """Returns a new dataset with each array indexed along the specified dimension(s). @@ -1378,15 +1378,19 @@ def isel(self, drop=False, **indexers): Parameters ---------- - drop : bool, optional - If ``drop=True``, drop coordinates variables indexed by integers - instead of making them scalar. - **indexers : {dim: indexer, ...} - Keyword arguments with names matching dimensions and values given + indexers : dict, optional + A dict with keys matching dimensions and values given by integers, slice objects or arrays. indexer can be a integer, slice, array-like or DataArray. If DataArrays are passed as indexers, xarray-style indexing will be carried out. See :ref:`indexing` for the details. + One of indexers or indexers_kwargs must be provided. + drop : bool, optional + If ``drop=True``, drop coordinates variables indexed by integers + instead of making them scalar. + **indexers_kwarg : {dim: indexer, ...}, optional + The keyword arguments form of ``indexers``. + One of indexers or indexers_kwargs must be provided. Returns ------- @@ -1404,12 +1408,15 @@ def isel(self, drop=False, **indexers): Dataset.sel DataArray.isel """ + + indexers = either_dict_or_kwargs(indexers, indexers_kwargs, 'isel') + indexers_list = self._validate_indexers(indexers) variables = OrderedDict() for name, var in iteritems(self._variables): var_indexers = {k: v for k, v in indexers_list if k in var.dims} - new_var = var.isel(**var_indexers) + new_var = var.isel(indexers=var_indexers) if not (drop and name in var_indexers): variables[name] = new_var @@ -1425,7 +1432,8 @@ def isel(self, drop=False, **indexers): .union(coord_vars)) return self._replace_vars_and_dims(variables, coord_names=coord_names) - def sel(self, method=None, tolerance=None, drop=False, **indexers): + def sel(self, indexers=None, method=None, tolerance=None, drop=False, + **indexers_kwargs): """Returns a new dataset with each array indexed by tick labels along the specified dimension(s). @@ -1444,6 +1452,14 @@ def sel(self, method=None, tolerance=None, drop=False, **indexers): Parameters ---------- + indexers : dict, optional + A dict with keys matching dimensions and values given + by scalars, slices or arrays of tick labels. For dimensions with + multi-index, the indexer may also be a dict-like object with keys + matching index level names. + If DataArrays are passed as indexers, xarray-style indexing will be + carried out. See :ref:`indexing` for the details. + One of indexers or indexers_kwargs must be provided. method : {None, 'nearest', 'pad'/'ffill', 'backfill'/'bfill'}, optional Method to use for inexact matches (requires pandas>=0.16): @@ -1459,13 +1475,9 @@ def sel(self, method=None, tolerance=None, drop=False, **indexers): drop : bool, optional If ``drop=True``, drop coordinates variables in `indexers` instead of making them scalar. - **indexers : {dim: indexer, ...} - Keyword arguments with names matching dimensions and values given - by scalars, slices or arrays of tick labels. For dimensions with - multi-index, the indexer may also be a dict-like object with keys - matching index level names. - If DataArrays are passed as indexers, xarray-style indexing will be - carried out. See :ref:`indexing` for the details. + **indexers_kwarg : {dim: indexer, ...}, optional + The keyword arguments form of ``indexers``. + One of indexers or indexers_kwargs must be provided. Returns ------- @@ -1484,9 +1496,10 @@ def sel(self, method=None, tolerance=None, drop=False, **indexers): Dataset.isel DataArray.sel """ - pos_indexers, new_indexes = remap_label_indexers(self, method, - tolerance, **indexers) - result = self.isel(drop=drop, **pos_indexers) + indexers = either_dict_or_kwargs(indexers, indexers_kwargs, 'sel') + pos_indexers, new_indexes = remap_label_indexers( + self, indexers=indexers, method=method, tolerance=tolerance) + result = self.isel(indexers=pos_indexers, drop=drop) return result._replace_indexes(new_indexes) def isel_points(self, dim='points', **indexers): @@ -1734,7 +1747,7 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): **indexers) def reindex(self, indexers=None, method=None, tolerance=None, copy=True, - **kw_indexers): + **indexers_kwargs): """Conform this object onto a new set of indexes, filling in missing values with NaN. @@ -1745,6 +1758,7 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, arrays of coordinates tick labels. Any mis-matched coordinate values will be filled in with NaN, and any mis-matched dimension names will simply be ignored. + One of indexers or indexers_kwargs must be provided. method : {None, 'nearest', 'pad'/'ffill', 'backfill'/'bfill'}, optional Method to use for filling index values in ``indexers`` not found in this dataset: @@ -1763,8 +1777,9 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, ``copy=False`` and reindexing is unnecessary, or can be performed with only slice operations, then the output may share memory with the input. In either case, a new xarray object is always returned. - **kw_indexers : optional + **indexers_kwarg : {dim: indexer, ...}, optional Keyword arguments in the same form as ``indexers``. + One of indexers or indexers_kwargs must be provided. Returns ------- @@ -1777,7 +1792,7 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, align pandas.Index.get_indexer """ - indexers = utils.combine_pos_and_kw_args(indexers, kw_indexers, + indexers = utils.either_dict_or_kwargs(indexers, indexers_kwargs, 'reindex') bad_dims = [d for d in indexers if d not in self.dims] diff --git a/xarray/core/utils.py b/xarray/core/utils.py index f6c5830cc9e..c3bb747fac5 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -185,7 +185,7 @@ def is_full_slice(value): return isinstance(value, slice) and value == slice(None) -def combine_pos_and_kw_args(pos_kwargs, kw_kwargs, func_name): +def either_dict_or_kwargs(pos_kwargs, kw_kwargs, func_name): if pos_kwargs is not None: if not is_dict_like(pos_kwargs): raise ValueError('the first argument to .%s must be a dictionary' diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 9dcb99459d4..52d470accfe 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -11,13 +11,13 @@ import xarray as xr # only for Dataset and DataArray from . import ( - arithmetic, common, dtypes, duck_array_ops, indexing, nputils, ops, utils,) + arithmetic, common, dtypes, duck_array_ops, indexing, nputils, ops, utils) from .indexing import ( BasicIndexer, OuterIndexer, PandasIndexAdapter, VectorizedIndexer, as_indexable) from .pycompat import ( OrderedDict, basestring, dask_array_type, integer_types, zip) -from .utils import OrderedSet +from .utils import OrderedSet, either_dict_or_kwargs try: import dask.array as da @@ -824,7 +824,7 @@ def chunk(self, chunks=None, name=None, lock=False): return type(self)(self.dims, data, self._attrs, self._encoding, fastpath=True) - def isel(self, **indexers): + def isel(self, indexers=None, drop=False, **indexers_kwargs): """Return a new array indexed along the specified dimension(s). Parameters @@ -841,6 +841,8 @@ def isel(self, **indexers): unless numpy fancy indexing was triggered by using an array indexer, in which case the data will be a copy. """ + indexers = either_dict_or_kwargs(indexers, indexers_kwargs, 'isel') + invalid = [k for k in indexers if k not in self.dims] if invalid: raise ValueError("dimensions %r do not exist" % invalid) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index a03d265c3e3..17e02fce7ed 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -17,8 +17,8 @@ from xarray.core.pycompat import OrderedDict, iteritems from xarray.tests import ( ReturnItem, TestCase, assert_allclose, assert_array_equal, assert_equal, - assert_identical, raises_regex, requires_bottleneck, requires_dask, - requires_scipy, source_ndarray, unittest, requires_cftime) + assert_identical, raises_regex, requires_bottleneck, requires_cftime, + requires_dask, requires_scipy, source_ndarray, unittest) class TestDataArray(TestCase): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 76e41c43c6d..38e2dce1633 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1435,7 +1435,7 @@ def test_sel_method(self): with raises_regex(TypeError, '``method``'): # this should not pass silently - data.sel(data) + data.sel(method=data) # cannot pass method if there is no associated coordinate with raises_regex(ValueError, 'cannot supply'): @@ -4181,6 +4181,12 @@ def test_dir_non_string(data_set): result = dir(data_set) assert not (5 in result) + # GH2172 + sample_data = np.random.uniform(size=[2, 2000, 10000]) + x = xr.Dataset({"sample_data": (sample_data.shape, sample_data)}) + x2 = x["sample_data"] + dir(x2) + def test_dir_unicode(data_set): data_set[u'unicode'] = 'uni' diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 1f73743d01d..ed8045b78e4 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -1,17 +1,21 @@ from __future__ import absolute_import, division, print_function +from datetime import datetime + import numpy as np import pandas as pd import pytest -from datetime import datetime from xarray.coding.cftimeindex import CFTimeIndex from xarray.core import duck_array_ops, utils from xarray.core.options import set_options from xarray.core.pycompat import OrderedDict +from xarray.core.utils import either_dict_or_kwargs + +from . import ( + TestCase, assert_array_equal, has_cftime, has_cftime_or_netCDF4, + requires_dask) from .test_coding_times import _all_cftime_date_types -from . import (TestCase, requires_dask, assert_array_equal, - has_cftime_or_netCDF4, has_cftime) class TestAlias(TestCase): @@ -245,3 +249,17 @@ def test_hidden_key_dict(): hkd[hidden_key] with pytest.raises(KeyError): del hkd[hidden_key] + + +def test_either_dict_or_kwargs(): + + result = either_dict_or_kwargs(dict(a=1), None, 'foo') + expected = dict(a=1) + assert result == expected + + result = either_dict_or_kwargs(None, dict(a=1), 'foo') + expected = dict(a=1) + assert result == expected + + with pytest.raises(ValueError, match=r'foo'): + result = either_dict_or_kwargs(dict(a=1), dict(a=1), 'foo') From fb7a43ea102c7706ad5d3bc8399264155cb273dd Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Mon, 28 May 2018 23:05:08 -0400 Subject: [PATCH 129/282] Rename takes kwargs (#2194) * rename takes kwargs * tests * better check for type of rename --- doc/whats-new.rst | 7 ++++++- xarray/core/dataarray.py | 17 +++++++++++------ xarray/core/dataset.py | 8 ++++++-- xarray/tests/test_dataarray.py | 3 +++ xarray/tests/test_dataset.py | 3 +++ 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 68bf5318bf5..f3af2b399d8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -49,11 +49,16 @@ Enhancements - :py:meth:`~DataArray.sel`, :py:meth:`~DataArray.isel` & :py:meth:`~DataArray.reindex`, (and their :py:class:`Dataset` counterparts) now support supplying a ``dict`` as a first argument, as an alternative to the existing approach - of supplying a set of `kwargs`. This allows for more robust behavior + of supplying `kwargs`. This allows for more robust behavior of dimension names which conflict with other keyword names, or are not strings. By `Maximilian Roos `_. +- :py:meth:`~DataArray.rename` now supports supplying `kwargs`, as an + alternative to the existing approach of supplying a ``dict`` as the + first argument. + By `Maximilian Roos `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index da9acb48a7a..01f7c91f3a5 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -19,8 +19,7 @@ from .options import OPTIONS from .pycompat import OrderedDict, basestring, iteritems, range, zip from .utils import ( - either_dict_or_kwargs, decode_numpy_dict_values, - ensure_us_time_resolution) + decode_numpy_dict_values, either_dict_or_kwargs, ensure_us_time_resolution) from .variable import ( IndexVariable, Variable, as_compatible_data, as_variable, assert_unique_multiindex_level_names) @@ -907,16 +906,20 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, indexers=indexers, method=method, tolerance=tolerance, copy=copy) return self._from_temp_dataset(ds) - def rename(self, new_name_or_name_dict): + def rename(self, new_name_or_name_dict=None, **names): """Returns a new DataArray with renamed coordinates or a new name. Parameters ---------- - new_name_or_name_dict : str or dict-like + new_name_or_name_dict : str or dict-like, optional If the argument is dict-like, it it used as a mapping from old names to new names for coordinates. Otherwise, use the argument as the new name for this array. + **names, optional + The keyword arguments form of a mapping from old names to + new names for coordinates. + One of new_name_or_name_dict or names must be provided. Returns @@ -929,8 +932,10 @@ def rename(self, new_name_or_name_dict): Dataset.rename DataArray.swap_dims """ - if utils.is_dict_like(new_name_or_name_dict): - dataset = self._to_temp_dataset().rename(new_name_or_name_dict) + if names or utils.is_dict_like(new_name_or_name_dict): + name_dict = either_dict_or_kwargs( + new_name_or_name_dict, names, 'rename') + dataset = self._to_temp_dataset().rename(name_dict) return self._from_temp_dataset(dataset) else: return self._replace(name=new_name_or_name_dict) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index d6a5ac1c172..08f5f70d72b 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1806,17 +1806,20 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, coord_names.update(indexers) return self._replace_vars_and_dims(variables, coord_names) - def rename(self, name_dict, inplace=False): + def rename(self, name_dict=None, inplace=False, **names): """Returns a new object with renamed variables and dimensions. Parameters ---------- - name_dict : dict-like + name_dict : dict-like, optional Dictionary whose keys are current variable or dimension names and whose values are the desired names. inplace : bool, optional If True, rename variables and dimensions in-place. Otherwise, return a new dataset object. + **names, optional + Keyword form of ``name_dict``. + One of name_dict or names must be provided. Returns ------- @@ -1828,6 +1831,7 @@ def rename(self, name_dict, inplace=False): Dataset.swap_dims DataArray.rename """ + name_dict = either_dict_or_kwargs(name_dict, names, 'rename') for k, v in name_dict.items(): if k not in self and k not in self.dims: raise ValueError("cannot rename %r because it is not a " diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 17e02fce7ed..ef9620e4dc4 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1270,6 +1270,9 @@ def test_rename(self): assert renamed.name == 'z' assert renamed.dims == ('z',) + renamed_kwargs = self.dv.x.rename(x='z').rename('z') + assert_identical(renamed, renamed_kwargs) + def test_swap_dims(self): array = DataArray(np.random.randn(3), {'y': ('x', list('abc'))}, 'x') expected = DataArray(array.values, {'y': list('abc')}, dims='y') diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 38e2dce1633..e64f9859c9e 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -1916,6 +1916,9 @@ def test_rename(self): with pytest.raises(UnexpectedDataAccess): renamed['renamed_var1'].values + renamed_kwargs = data.rename(**newnames) + assert_identical(renamed, renamed_kwargs) + def test_rename_old_name(self): # regtest for GH1477 data = create_test_data() From 5ddfee6b194f8bdce8bb42cddfabe3e1c142ef16 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 28 May 2018 20:15:07 -0700 Subject: [PATCH 130/282] Fix DataArray.groupby().reduce() mutating input array (#2169) * Fix DataArray.groupby().reduce() mutating input array Fixes GH2153 * Fix test failure --- doc/whats-new.rst | 5 +++++ xarray/core/dataarray.py | 2 +- xarray/tests/test_computation.py | 2 +- xarray/tests/test_groupby.py | 11 +++++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index f3af2b399d8..1465b9e68a5 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -82,6 +82,11 @@ Bug fixes the index) (:issue:`2165`). By `Spencer Clark `_. +- Fix ``DataArray.groupby().reduce()`` mutating coordinates on the input array + when grouping over dimension coordinates with duplicated entries + (:issue:`2153`). + By `Stephan Hoyer `_ + - Fix Dataset.to_netcdf() cannot create group with engine="h5netcdf" (:issue:`2177`). By `Stephan Hoyer `_ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 01f7c91f3a5..fd2b49cc08a 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -252,7 +252,7 @@ def _replace(self, variable=None, coords=None, name=__default): def _replace_maybe_drop_dims(self, variable, name=__default): if variable.dims == self.dims: - coords = None + coords = self._coords.copy() else: allowed_dims = set(variable.dims) coords = OrderedDict((k, v) for k, v in self._coords.items() diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index c829453cc9d..bbcc02baf5a 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -553,7 +553,7 @@ def test_apply_dask(): array = da.ones((2,), chunks=2) variable = xr.Variable('x', array) coords = xr.DataArray(variable).coords.variables - data_array = xr.DataArray(variable, coords, fastpath=True) + data_array = xr.DataArray(variable, dims=['x'], coords=coords) dataset = xr.Dataset({'y': variable}) # encountered dask array, but did not set dask='allowed' diff --git a/xarray/tests/test_groupby.py b/xarray/tests/test_groupby.py index fd53e410583..6dd14f5d6ad 100644 --- a/xarray/tests/test_groupby.py +++ b/xarray/tests/test_groupby.py @@ -5,6 +5,7 @@ import pytest import xarray as xr +from . import assert_identical from xarray.core.groupby import _consolidate_slices @@ -73,4 +74,14 @@ def test_groupby_duplicate_coordinate_labels(): assert expected.equals(actual) +def test_groupby_input_mutation(): + # regression test for GH2153 + array = xr.DataArray([1, 2, 3], [('x', [2, 2, 1])]) + array_copy = array.copy() + expected = xr.DataArray([3, 3], [('x', [1, 2])]) + actual = array.groupby('x').sum() + assert_identical(expected, actual) + assert_identical(array, array_copy) # should not modify inputs + + # TODO: move other groupby tests from test_dataset and test_dataarray over here From 9c8005937556211a8bf28a946744da3768846c5a Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 28 May 2018 21:34:46 -0700 Subject: [PATCH 131/282] Test suite: explicitly ignore irrelevant warnings (#2162) * Test suite: explicitly ignore irrelevant warnings Also includes a fix for Dataset.update() * Cleaner implementation of Dataset.__setitem__ * more lint * Fix dask version check * Fix warning in Dataset.update() and clean-up logic * Fix whats new * More whats new * DeprecationWarning -> FutureWarning for old resample API --- doc/whats-new.rst | 9 +++++-- xarray/conventions.py | 2 +- xarray/core/common.py | 2 +- xarray/core/duck_array_ops.py | 20 +++++++++----- xarray/core/merge.py | 23 ++++++++++------- xarray/tests/__init__.py | 10 +++++-- xarray/tests/test_backends.py | 42 ++++++++++++++++++------------ xarray/tests/test_coding_times.py | 4 ++- xarray/tests/test_conventions.py | 4 ++- xarray/tests/test_dask.py | 3 +-- xarray/tests/test_dataarray.py | 30 +++++++++++++-------- xarray/tests/test_dataset.py | 43 ++++++++++++++++++++++++------- xarray/tests/test_variable.py | 7 +++-- 13 files changed, 132 insertions(+), 67 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 1465b9e68a5..aef80a2b30a 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -62,10 +62,15 @@ Enhancements Bug fixes ~~~~~~~~~ -- Fixed a bug where `to_netcdf(..., unlimited_dims='bar'` yielded NetCDF files - with spurious 0-length dimensions (i.e. `b`, `a`, and `r`) (:issue:`2134`). +- Fixed a bug where ``to_netcdf(..., unlimited_dims='bar')`` yielded NetCDF + files with spurious 0-length dimensions (i.e. ``b``, ``a``, and ``r``) + (:issue:`2134`). By `Joe Hamman `_. +- Removed spurious warnings with ``Dataset.update(Dataset)`` (:issue:`2161`) + and ``array.equals(array)`` when ``array`` contains ``NaT`` (:issue:`2162`). + By `Stephan Hoyer `_. + - Aggregations with :py:meth:`Dataset.reduce` (including ``mean``, ``sum``, etc) no longer drop unrelated coordinates (:issue:`1470`). Also fixed a bug where non-scalar data-variables that did not include the aggregation diff --git a/xarray/conventions.py b/xarray/conventions.py index ed90c34387b..6171c353a0d 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -89,7 +89,7 @@ def maybe_encode_nonstring_dtype(var, name=None): warnings.warn('saving variable %s with floating ' 'point data as an integer dtype without ' 'any _FillValue to use for NaNs' % name, - SerializationWarning, stacklevel=3) + SerializationWarning, stacklevel=10) data = duck_array_ops.around(data)[...] data = data.astype(dtype=dtype) var = Variable(dims, data, attrs, encoding) diff --git a/xarray/core/common.py b/xarray/core/common.py index f623091ebdb..d69c60eed56 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -696,7 +696,7 @@ def _resample_immediately(self, freq, dim, how, skipna, "how=\"{how}\", instead consider using " ".resample({dim}=\"{freq}\").{how}('{dim}') ".format( dim=dim, freq=freq, how=how), - DeprecationWarning, stacklevel=3) + FutureWarning, stacklevel=3) if isinstance(dim, basestring): dim = self[dim] diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 69b0d0825be..065ac165a0d 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -145,10 +145,13 @@ def array_equiv(arr1, arr2): if arr1.shape != arr2.shape: return False - flag_array = (arr1 == arr2) - flag_array |= (isnull(arr1) & isnull(arr2)) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', "In the future, 'NAT == x'") - return bool(flag_array.all()) + flag_array = (arr1 == arr2) + flag_array |= (isnull(arr1) & isnull(arr2)) + + return bool(flag_array.all()) def array_notnull_equiv(arr1, arr2): @@ -159,11 +162,14 @@ def array_notnull_equiv(arr1, arr2): if arr1.shape != arr2.shape: return False - flag_array = (arr1 == arr2) - flag_array |= isnull(arr1) - flag_array |= isnull(arr2) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', "In the future, 'NAT == x'") + + flag_array = (arr1 == arr2) + flag_array |= isnull(arr1) + flag_array |= isnull(arr2) - return bool(flag_array.all()) + return bool(flag_array.all()) def count(data, axis=None): diff --git a/xarray/core/merge.py b/xarray/core/merge.py index d3c9871abef..40a58d6e84e 100644 --- a/xarray/core/merge.py +++ b/xarray/core/merge.py @@ -547,21 +547,24 @@ def dataset_merge_method(dataset, other, overwrite_vars, compat, join): def dataset_update_method(dataset, other): - """Guts of the Dataset.update method + """Guts of the Dataset.update method. - This drops a duplicated coordinates from `other` (GH:2068) + This drops a duplicated coordinates from `other` if `other` is not an + `xarray.Dataset`, e.g., if it's a dict with DataArray values (GH2068, + GH2180). """ from .dataset import Dataset from .dataarray import DataArray - other = other.copy() - for k, obj in other.items(): - if isinstance(obj, (Dataset, DataArray)): - # drop duplicated coordinates - coord_names = [c for c in obj.coords - if c not in obj.dims and c in dataset.coords] - if coord_names: - other[k] = obj.drop(coord_names) + if not isinstance(other, Dataset): + other = OrderedDict(other) + for key, value in other.items(): + if isinstance(value, DataArray): + # drop conflicting coordinates + coord_names = [c for c in value.coords + if c not in value.dims and c in dataset.coords] + if coord_names: + other[key] = value.drop(coord_names) return merge_core([dataset, other], priority_arg=1, indexes=dataset.indexes) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 7584ed79a06..3acd26235ce 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -87,7 +87,10 @@ def _importorskip(modname, minversion=None): has_pathlib, requires_pathlib = _importorskip('pathlib2') if has_dask: import dask - dask.set_options(get=dask.get) + if LooseVersion(dask.__version__) < '0.18': + dask.set_options(get=dask.get) + else: + dask.config.set(scheduler='sync') try: import_seaborn() has_seaborn = True @@ -191,7 +194,10 @@ def source_ndarray(array): """Given an ndarray, return the base object which holds its memory, or the object itself. """ - base = getattr(array, 'base', np.asarray(array).base) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'DatetimeIndex.base') + warnings.filterwarnings('ignore', 'TimedeltaIndex.base') + base = getattr(array, 'base', np.asarray(array).base) if base is None: base = array return base diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 0768a942a77..b80cb18e2be 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -363,21 +363,26 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): expected_decoded_t0 = np.array([date_type(1, 1, 1)]) expected_calendar = times[0].calendar - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected, save_kwargs=kwds) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t.encoding['units'] == - 'days since 0001-01-01 00:00:00.000000') - assert (actual.t.encoding['calendar'] == - expected_calendar) - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t0.encoding['units'] == - 'days since 0001-01-01') - assert (actual.t.encoding['calendar'] == - expected_calendar) + with warnings.catch_warnings(): + if expected_calendar in {'proleptic_gregorian', 'gregorian'}: + warnings.filterwarnings( + 'ignore', 'Unable to decode time axis') + + with xr.set_options(enable_cftimeindex=True): + with self.roundtrip(expected, save_kwargs=kwds) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t.encoding['units'] == + 'days since 0001-01-01 00:00:00.000000') + assert (actual.t.encoding['calendar'] == + expected_calendar) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t0.encoding['units'] == + 'days since 0001-01-01') + assert (actual.t.encoding['calendar'] == + expected_calendar) def test_roundtrip_timedelta_data(self): time_deltas = pd.to_timedelta(['1h', '2h', 'NaT']) @@ -767,8 +772,11 @@ def test_default_fill_value(self): # Test default encoding for int: ds = Dataset({'x': ('y', np.arange(10.0))}) kwargs = dict(encoding={'x': {'dtype': 'int16'}}) - with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertTrue('_FillValue' not in actual.x.encoding) + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', '.*floating point data as an integer') + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + self.assertTrue('_FillValue' not in actual.x.encoding) self.assertEqual(ds.x.encoding, {}) # Test default encoding for implicit int: diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 6329e91ac78..4d6ca731bb2 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -130,7 +130,9 @@ def test_decode_cf_datetime_overflow(): expected = (datetime(1677, 12, 31), datetime(2262, 4, 12)) for i, day in enumerate(days): - result = coding.times.decode_cf_datetime(day, units) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + result = coding.times.decode_cf_datetime(day, units) assert result == expected[i] diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 62ff8d7ee1a..acc1c978579 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -219,7 +219,9 @@ def test_decode_cf_datetime_transition_to_invalid(self): ds = Dataset(coords={'time': [0, 266 * 365]}) units = 'days since 2000-01-01 00:00:00' ds.time.attrs = dict(units=units) - ds_decoded = conventions.decode_cf(ds) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'unable to decode time') + ds_decoded = conventions.decode_cf(ds) expected = [datetime(2000, 1, 1, 0, 0), datetime(2265, 10, 28, 0, 0)] diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 1e4f313897b..ee5b3514348 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -459,8 +459,7 @@ def counting_get(*args, **kwargs): count[0] += 1 return dask.get(*args, **kwargs) - with dask.set_options(get=counting_get): - ds.load() + ds.load(get=counting_get) assert count[0] == 1 def test_stack(self): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index ef9620e4dc4..d339e6402b6 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -4,6 +4,7 @@ from copy import deepcopy from distutils.version import LooseVersion from textwrap import dedent +import warnings import numpy as np import pandas as pd @@ -321,11 +322,14 @@ def test_constructor_from_self_described(self): actual = DataArray(series) assert_equal(expected[0].reset_coords('x', drop=True), actual) - panel = pd.Panel({0: frame}) - actual = DataArray(panel) - expected = DataArray([data], expected.coords, ['dim_0', 'x', 'y']) - expected['dim_0'] = [0] - assert_identical(expected, actual) + if hasattr(pd, 'Panel'): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', r'\W*Panel is deprecated') + panel = pd.Panel({0: frame}) + actual = DataArray(panel) + expected = DataArray([data], expected.coords, ['dim_0', 'x', 'y']) + expected['dim_0'] = [0] + assert_identical(expected, actual) expected = DataArray(data, coords={'x': ['a', 'b'], 'y': [-1, -2], @@ -2320,7 +2324,7 @@ def test_resample_old_vs_new_api(self): array = DataArray(np.ones(10), [('time', times)]) # Simple mean - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_mean = array.resample('1D', 'time', how='mean') new_mean = array.resample(time='1D').mean() assert_identical(old_mean, new_mean) @@ -2329,7 +2333,7 @@ def test_resample_old_vs_new_api(self): attr_array = array.copy() attr_array.attrs['meta'] = 'data' - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_mean = attr_array.resample('1D', dim='time', how='mean', keep_attrs=True) new_mean = attr_array.resample(time='1D').mean(keep_attrs=True) @@ -2340,7 +2344,7 @@ def test_resample_old_vs_new_api(self): nan_array = array.copy() nan_array[1] = np.nan - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_mean = nan_array.resample('1D', 'time', how='mean', skipna=False) new_mean = nan_array.resample(time='1D').mean(skipna=False) @@ -2354,12 +2358,12 @@ def test_resample_old_vs_new_api(self): # Discard attributes on the call using the new api to match # convention from old api new_api = getattr(resampler, method)(keep_attrs=False) - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_api = array.resample('1D', dim='time', how=method) assert_identical(new_api, old_api) for method in [np.mean, np.sum, np.max, np.min]: new_api = resampler.reduce(method) - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_api = array.resample('1D', dim='time', how=method) assert_identical(new_api, old_api) @@ -2713,9 +2717,13 @@ def test_to_pandas(self): # roundtrips for shape in [(3,), (3, 4), (3, 4, 5)]: + if len(shape) > 2 and not hasattr(pd, 'Panel'): + continue dims = list('abc')[:len(shape)] da = DataArray(np.random.randn(*shape), dims=dims) - roundtripped = DataArray(da.to_pandas()).drop(dims) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', r'\W*Panel is deprecated') + roundtripped = DataArray(da.to_pandas()).drop(dims) assert_identical(da, roundtripped) with raises_regex(ValueError, 'cannot convert'): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index e64f9859c9e..2dad40ae8f6 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -5,6 +5,7 @@ from distutils.version import LooseVersion from io import StringIO from textwrap import dedent +import warnings import numpy as np import pandas as pd @@ -338,14 +339,20 @@ def test_constructor_pandas_single(self): das = [ DataArray(np.random.rand(4), dims=['a']), # series DataArray(np.random.rand(4, 3), dims=['a', 'b']), # df - DataArray(np.random.rand(4, 3, 2), dims=['a', 'b', 'c']), # panel ] - for a in das: - pandas_obj = a.to_pandas() - ds_based_on_pandas = Dataset(pandas_obj) - for dim in ds_based_on_pandas.data_vars: - assert_array_equal(ds_based_on_pandas[dim], pandas_obj[dim]) + if hasattr(pd, 'Panel'): + das.append( + DataArray(np.random.rand(4, 3, 2), dims=['a', 'b', 'c'])) + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', r'\W*Panel is deprecated') + for a in das: + pandas_obj = a.to_pandas() + ds_based_on_pandas = Dataset(pandas_obj) + for dim in ds_based_on_pandas.data_vars: + assert_array_equal( + ds_based_on_pandas[dim], pandas_obj[dim]) def test_constructor_compat(self): data = OrderedDict([('x', DataArray(0, coords={'y': 1})), @@ -2139,6 +2146,22 @@ def test_update(self): actual.update(other) assert_identical(expected, actual) + def test_update_overwrite_coords(self): + data = Dataset({'a': ('x', [1, 2])}, {'b': 3}) + data.update(Dataset(coords={'b': 4})) + expected = Dataset({'a': ('x', [1, 2])}, {'b': 4}) + assert_identical(data, expected) + + data = Dataset({'a': ('x', [1, 2])}, {'b': 3}) + data.update(Dataset({'c': 5}, coords={'b': 4})) + expected = Dataset({'a': ('x', [1, 2]), 'c': 5}, {'b': 4}) + assert_identical(data, expected) + + data = Dataset({'a': ('x', [1, 2])}, {'b': 3}) + data.update({'c': DataArray(5, coords={'b': 4})}) + expected = Dataset({'a': ('x', [1, 2]), 'c': 5}, {'b': 3}) + assert_identical(data, expected) + def test_update_auto_align(self): ds = Dataset({'x': ('t', [3, 4])}, {'t': [0, 1]}) @@ -2343,14 +2366,14 @@ def test_setitem_with_coords(self): actual = ds.copy() actual['var3'] = other assert_identical(expected, actual) - assert 'numbers' in other # should not change other + assert 'numbers' in other.coords # should not change other # with alignment other = ds['var3'].isel(dim3=slice(1, -1)) other['numbers'] = ('dim3', np.arange(8)) actual = ds.copy() actual['var3'] = other - assert 'numbers' in other # should not change other + assert 'numbers' in other.coords # should not change other expected = ds.copy() expected['var3'] = ds['var3'].isel(dim3=slice(1, -1)) assert_identical(expected, actual) @@ -2362,7 +2385,7 @@ def test_setitem_with_coords(self): actual = ds.copy() actual['var3'] = other assert 'position' in actual - assert 'position' in other + assert 'position' in other.coords # assigning a coordinate-only dataarray actual = ds.copy() @@ -2774,7 +2797,7 @@ def test_resample_old_vs_new_api(self): # Discard attributes on the call using the new api to match # convention from old api new_api = getattr(resampler, method)(keep_attrs=False) - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_api = ds.resample('1D', dim='time', how=method) assert_identical(new_api, old_api) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 722d1af14f7..c486a394ae6 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta from distutils.version import LooseVersion from textwrap import dedent +import warnings import numpy as np import pandas as pd @@ -138,8 +139,10 @@ def _assertIndexedLikeNDArray(self, variable, expected_value0, assert variable.equals(variable.copy()) assert variable.identical(variable.copy()) # check value is equal for both ndarray and Variable - assert variable.values[0] == expected_value0 - assert variable[0].values == expected_value0 + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', "In the future, 'NAT == x'") + assert variable.values[0] == expected_value0 + assert variable[0].values == expected_value0 # check type or dtype is consistent for both ndarray and Variable if expected_dtype is None: # check output type instead of array dtype From 7036eb5b629f2112da9aa13538aecb07f0f83f5a Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 30 May 2018 20:22:58 -0400 Subject: [PATCH 132/282] Align DataArrays based on coords in Dataset constructor (#1826) * Align da based on explicit coords * add tags to gitignore * random formatting spot * initial tests * apply the test to the right degree of freedom - the coords of the variable added in * couple more for gitignore * @stickler-ci doesn't like `range` * more tests * more gitignores * whats new * raise * message= * Add all testmon files to gitignore * cast single dim tuples to indexes * test on different dataset coords types * updated whatsnew * version from Stephan's feedback; works but not clean * I think much cleaner version * formatting --- .gitignore | 1 + doc/whats-new.rst | 5 +++++ xarray/core/merge.py | 12 ++++++++++- xarray/tests/test_dataset.py | 41 +++++++++++++++++++++++++++++++++++- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 92e488ed616..2a016bb9228 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ nosetests.xml .ropeproject/ .tags* .testmon* +.tmontmp/ .pytest_cache dask-worker-space/ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index aef80a2b30a..37b022f45c8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -46,6 +46,11 @@ Enhancements to manage its version strings. (:issue:`1300`). By `Joe Hamman `_. +- `:py:class:`Dataset`s align `:py:class:`DataArray`s to coords that are explicitly + passed into the constructor, where previously an error would be raised. + (:issue:`674`) + By `Maximilian Roos Date: Thu, 31 May 2018 08:40:03 -0700 Subject: [PATCH 133/282] Validate output dimension sizes with apply_ufunc (#2155) * Validate output dimension sizes with apply_ufunc Fixes GH1931 Uses of apply_ufunc that change dimension size now raise an explicit error, e.g., >>> xr.apply_ufunc(lambda x: x[:5], xr.Variable('x', np.arange(10))) ValueError: size of dimension 'x' on inputs was unexpectedly changed by applied function from 10 to 5. Only dimensions specified in ``exclude_dims`` with xarray.apply_ufunc are allowed to change size. * lint * More output validation for apply_ufunc --- doc/whats-new.rst | 4 ++ xarray/core/computation.py | 90 ++++++++++++++++++++++++-------- xarray/tests/test_computation.py | 88 +++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 22 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 37b022f45c8..f3db96f99bf 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -67,6 +67,10 @@ Enhancements Bug fixes ~~~~~~~~~ +- :py:func:`apply_ufunc` now directly validates output variables + (:issue:`1931`). + By `Stephan Hoyer `_. + - Fixed a bug where ``to_netcdf(..., unlimited_dims='bar')`` yielded NetCDF files with spurious 0-length dimensions (i.e. ``b``, ``a``, and ``r``) (:issue:`2134`). diff --git a/xarray/core/computation.py b/xarray/core/computation.py index f6e22dfe6c1..ebbce114ec3 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -513,7 +513,7 @@ def broadcast_compat_data(variable, broadcast_dims, core_dims): def apply_variable_ufunc(func, *args, **kwargs): """apply_variable_ufunc(func, *args, signature, exclude_dims=frozenset()) """ - from .variable import Variable + from .variable import Variable, as_compatible_data signature = kwargs.pop('signature') exclude_dims = kwargs.pop('exclude_dims', _DEFAULT_FROZEN_SET) @@ -559,20 +559,42 @@ def func(*arrays): 'apply_ufunc: {}'.format(dask)) result_data = func(*input_data) - if signature.num_outputs > 1: - output = [] - for dims, data in zip(output_dims, result_data): - var = Variable(dims, data) - if keep_attrs and isinstance(args[0], Variable): - var.attrs.update(args[0].attrs) - output.append(var) - return tuple(output) - else: - dims, = output_dims - var = Variable(dims, result_data) + if signature.num_outputs == 1: + result_data = (result_data,) + elif (not isinstance(result_data, tuple) or + len(result_data) != signature.num_outputs): + raise ValueError('applied function does not have the number of ' + 'outputs specified in the ufunc signature. ' + 'Result is not a tuple of {} elements: {!r}' + .format(signature.num_outputs, result_data)) + + output = [] + for dims, data in zip(output_dims, result_data): + data = as_compatible_data(data) + if data.ndim != len(dims): + raise ValueError( + 'applied function returned data with unexpected ' + 'number of dimensions: {} vs {}, for dimensions {}' + .format(data.ndim, len(dims), dims)) + + var = Variable(dims, data, fastpath=True) + for dim, new_size in var.sizes.items(): + if dim in dim_sizes and new_size != dim_sizes[dim]: + raise ValueError( + 'size of dimension {!r} on inputs was unexpectedly ' + 'changed by applied function from {} to {}. Only ' + 'dimensions specified in ``exclude_dims`` with ' + 'xarray.apply_ufunc are allowed to change size.' + .format(dim, dim_sizes[dim], new_size)) + if keep_attrs and isinstance(args[0], Variable): var.attrs.update(args[0].attrs) - return var + output.append(var) + + if signature.num_outputs == 1: + return output[0] + else: + return tuple(output) def _apply_with_dask_atop(func, args, input_dims, output_dims, signature, @@ -719,7 +741,8 @@ def apply_ufunc(func, *args, **kwargs): Core dimensions on the inputs to exclude from alignment and broadcasting entirely. Any input coordinates along these dimensions will be dropped. Each excluded dimension must also appear in - ``input_core_dims`` for at least one argument. + ``input_core_dims`` for at least one argument. Only dimensions listed + here are allowed to change size between input and output objects. vectorize : bool, optional If True, then assume ``func`` only takes arrays defined over core dimensions as input and vectorize it automatically with @@ -777,15 +800,38 @@ def apply_ufunc(func, *args, **kwargs): Examples -------- - For illustrative purposes only, here are examples of how you could use - ``apply_ufunc`` to write functions to (very nearly) replicate existing - xarray functionality: - Calculate the vector magnitude of two arguments:: + Calculate the vector magnitude of two arguments: + + >>> def magnitude(a, b): + ... func = lambda x, y: np.sqrt(x ** 2 + y ** 2) + ... return xr.apply_ufunc(func, a, b) + + You can now apply ``magnitude()`` to ``xr.DataArray`` and ``xr.Dataset`` + objects, with automatically preserved dimensions and coordinates, e.g., + + >>> array = xr.DataArray([1, 2, 3], coords=[('x', [0.1, 0.2, 0.3])]) + >>> magnitude(array, -array) + + array([1.414214, 2.828427, 4.242641]) + Coordinates: + * x (x) float64 0.1 0.2 0.3 + + Plain scalars, numpy arrays and a mix of these with xarray objects is also + supported: + + >>> magnitude(4, 5) + 5.0 + >>> magnitude(3, np.array([0, 4])) + array([3., 5.]) + >>> magnitude(array, 0) + + array([1., 2., 3.]) + Coordinates: + * x (x) float64 0.1 0.2 0.3 - def magnitude(a, b): - func = lambda x, y: np.sqrt(x ** 2 + y ** 2) - return xr.apply_func(func, a, b) + Other examples of how you could use ``apply_ufunc`` to write functions to + (very nearly) replicate existing xarray functionality: Compute the mean (``.mean``) over one dimension:: @@ -795,7 +841,7 @@ def mean(obj, dim): input_core_dims=[[dim]], kwargs={'axis': -1}) - Inner product over a specific dimension:: + Inner product over a specific dimension (like ``xr.dot``):: def _inner(x, y): result = np.matmul(x[..., np.newaxis, :], y[..., :, np.newaxis]) diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index bbcc02baf5a..37f97a81f82 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -752,6 +752,94 @@ def test_vectorize_dask(): assert_identical(expected, actual) +def test_output_wrong_number(): + variable = xr.Variable('x', np.arange(10)) + + def identity(x): + return x + + def tuple3x(x): + return (x, x, x) + + with raises_regex(ValueError, 'number of outputs'): + apply_ufunc(identity, variable, output_core_dims=[(), ()]) + + with raises_regex(ValueError, 'number of outputs'): + apply_ufunc(tuple3x, variable, output_core_dims=[(), ()]) + + +def test_output_wrong_dims(): + variable = xr.Variable('x', np.arange(10)) + + def add_dim(x): + return x[..., np.newaxis] + + def remove_dim(x): + return x[..., 0] + + with raises_regex(ValueError, 'unexpected number of dimensions'): + apply_ufunc(add_dim, variable, output_core_dims=[('y', 'z')]) + + with raises_regex(ValueError, 'unexpected number of dimensions'): + apply_ufunc(add_dim, variable) + + with raises_regex(ValueError, 'unexpected number of dimensions'): + apply_ufunc(remove_dim, variable) + + +def test_output_wrong_dim_size(): + array = np.arange(10) + variable = xr.Variable('x', array) + data_array = xr.DataArray(variable, [('x', -array)]) + dataset = xr.Dataset({'y': variable}, {'x': -array}) + + def truncate(array): + return array[:5] + + def apply_truncate_broadcast_invalid(obj): + return apply_ufunc(truncate, obj) + + with raises_regex(ValueError, 'size of dimension'): + apply_truncate_broadcast_invalid(variable) + with raises_regex(ValueError, 'size of dimension'): + apply_truncate_broadcast_invalid(data_array) + with raises_regex(ValueError, 'size of dimension'): + apply_truncate_broadcast_invalid(dataset) + + def apply_truncate_x_x_invalid(obj): + return apply_ufunc(truncate, obj, input_core_dims=[['x']], + output_core_dims=[['x']]) + + with raises_regex(ValueError, 'size of dimension'): + apply_truncate_x_x_invalid(variable) + with raises_regex(ValueError, 'size of dimension'): + apply_truncate_x_x_invalid(data_array) + with raises_regex(ValueError, 'size of dimension'): + apply_truncate_x_x_invalid(dataset) + + def apply_truncate_x_z(obj): + return apply_ufunc(truncate, obj, input_core_dims=[['x']], + output_core_dims=[['z']]) + + assert_identical(xr.Variable('z', array[:5]), + apply_truncate_x_z(variable)) + assert_identical(xr.DataArray(array[:5], dims=['z']), + apply_truncate_x_z(data_array)) + assert_identical(xr.Dataset({'y': ('z', array[:5])}), + apply_truncate_x_z(dataset)) + + def apply_truncate_x_x_valid(obj): + return apply_ufunc(truncate, obj, input_core_dims=[['x']], + output_core_dims=[['x']], exclude_dims={'x'}) + + assert_identical(xr.Variable('x', array[:5]), + apply_truncate_x_x_valid(variable)) + assert_identical(xr.DataArray(array[:5], dims=['x']), + apply_truncate_x_x_valid(data_array)) + assert_identical(xr.Dataset({'y': ('x', array[:5])}), + apply_truncate_x_x_valid(dataset)) + + @pytest.mark.parametrize('use_dask', [True, False]) def test_dot(use_dask): if use_dask: From 9d60897a6544d3a2d4b9b3b64008b2bc316d8b98 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 1 Jun 2018 10:01:33 +0900 Subject: [PATCH 134/282] Support dot with older dask (#2205) * Support dot with older dask * add an if block for non-dask environment --- doc/whats-new.rst | 12 ++++++++---- xarray/core/computation.py | 13 +++++++++++++ xarray/tests/test_computation.py | 8 +++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index f3db96f99bf..e3c4b050812 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -37,6 +37,10 @@ Documentation Enhancements ~~~~~~~~~~~~ +- `:py:meth:`~DataArray.dot` and :py:func:`~dot` are partly supported with older + dask<0.17.4. (related to :issue:`2203`) + By `Keisuke Fujii `_. - :py:meth:`~DataArray.rename` now supports supplying `kwargs`, as an - alternative to the existing approach of supplying a ``dict`` as the + alternative to the existing approach of supplying a ``dict`` as the first argument. By `Maximilian Roos `_. @@ -100,7 +104,7 @@ Bug fixes when grouping over dimension coordinates with duplicated entries (:issue:`2153`). By `Stephan Hoyer `_ - + - Fix Dataset.to_netcdf() cannot create group with engine="h5netcdf" (:issue:`2177`). By `Stephan Hoyer `_ diff --git a/xarray/core/computation.py b/xarray/core/computation.py index ebbce114ec3..6a49610cb7b 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -1061,6 +1061,19 @@ def dot(*arrays, **kwargs): output_core_dims = [tuple(d for d in all_dims if d not in dims + broadcast_dims)] + # older dask than 0.17.4, we use tensordot if possible. + if isinstance(arr.data, dask_array_type): + import dask + if LooseVersion(dask.__version__) < LooseVersion('0.17.4'): + if len(broadcast_dims) == 0 and len(arrays) == 2: + axes = [[arr.get_axis_num(d) for d in arr.dims if d in dims] + for arr in arrays] + return apply_ufunc(duck_array_ops.tensordot, *arrays, + dask='allowed', + input_core_dims=input_core_dims, + output_core_dims=output_core_dims, + kwargs={'axes': axes}) + # construct einsum subscripts, such as '...abc,...ab->...c' # Note: input_core_dims are always moved to the last position subscripts_list = ['...' + ''.join([dim_map[d] for d in ds]) for ds diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index 37f97a81f82..a802b91a3db 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -845,9 +845,6 @@ def test_dot(use_dask): if use_dask: if not has_dask: pytest.skip('test for dask.') - import dask - if LooseVersion(dask.__version__) < LooseVersion('0.17.3'): - pytest.skip("needs dask.array.einsum") a = np.arange(30 * 4).reshape(30, 4) b = np.arange(30 * 4 * 5).reshape(30, 4, 5) @@ -872,6 +869,11 @@ def test_dot(use_dask): assert (actual.data == np.einsum('ij,ijk->k', a, b)).all() assert isinstance(actual.variable.data, type(da_a.variable.data)) + if use_dask: + import dask + if LooseVersion(dask.__version__) < LooseVersion('0.17.3'): + pytest.skip("needs dask.array.einsum") + # for only a single array is passed without dims argument, just return # as is actual = xr.dot(da_a) From 4106b949091d06f96ac3c1d07e95917f235bfb5f Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 31 May 2018 18:09:38 -0700 Subject: [PATCH 135/282] Fix dtype=S1 encoding in to_netcdf() (#2158) * Fix dtype=S1 encoding in to_netcdf() Fixes GH2149 * Add test_encoding_kwarg_compression from crusaderky * Fix dtype=S1 in kwargs for bytes, too * Fix lint * Move compression encoding kwarg test * Remvoe no longer relevant chanegs * Fix encoding dtype=str * More lint * Fix failed tests * Review comments * oops, we still need to skip that test * check for presence in a tuple rather than making two comparisons --- doc/whats-new.rst | 5 +++ xarray/backends/h5netcdf_.py | 19 +++++++++-- xarray/backends/netCDF4_.py | 27 ++++++++++++--- xarray/coding/strings.py | 7 ++-- xarray/conventions.py | 10 ++---- xarray/tests/test_backends.py | 57 ++++++++++++++++++++++++++++++-- xarray/tests/test_conventions.py | 7 ++++ 7 files changed, 114 insertions(+), 18 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e3c4b050812..c4c8db243d4 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -71,6 +71,11 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed a regression in 0.10.4, where explicitly specifying ``dtype='S1'`` or + ``dtype=str`` in ``encoding`` with ``to_netcdf()`` raised an error + (:issue:`2149`). + `Stephan Hoyer `_ + - :py:func:`apply_ufunc` now directly validates output variables (:issue:`1931`). By `Stephan Hoyer `_. diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 6b3cd9ebb15..ecc83e98691 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -94,6 +94,8 @@ def __init__(self, filename, mode='r', format=None, group=None, super(H5NetCDFStore, self).__init__(writer, lock=lock) def open_store_variable(self, name, var): + import h5py + with self.ensure_open(autoclose=False): dimensions = var.dimensions data = indexing.LazilyOuterIndexedArray( @@ -119,6 +121,15 @@ def open_store_variable(self, name, var): encoding['source'] = self._filename encoding['original_shape'] = var.shape + vlen_dtype = h5py.check_dtype(vlen=var.dtype) + if vlen_dtype is unicode_type: + encoding['dtype'] = str + elif vlen_dtype is not None: # pragma: no cover + # xarray doesn't support writing arbitrary vlen dtypes yet. + pass + else: + encoding['dtype'] = var.dtype + return Variable(dimensions, data, attrs, encoding) def get_variables(self): @@ -161,7 +172,8 @@ def prepare_variable(self, name, variable, check_encoding=False, import h5py attrs = variable.attrs.copy() - dtype = _get_datatype(variable) + dtype = _get_datatype( + variable, raise_on_invalid_encoding=check_encoding) fillvalue = attrs.pop('_FillValue', None) if dtype is str and fillvalue is not None: @@ -189,8 +201,9 @@ def prepare_variable(self, name, variable, check_encoding=False, raise ValueError("'zlib' and 'compression' encodings mismatch") encoding.setdefault('compression', 'gzip') - if (check_encoding and encoding.get('complevel') not in - (None, encoding.get('compression_opts'))): + if (check_encoding and + 'complevel' in encoding and 'compression_opts' in encoding and + encoding['complevel'] != encoding['compression_opts']): raise ValueError("'complevel' and 'compression_opts' encodings " "mismatch") complevel = encoding.pop('complevel', 0) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 5391a890fb3..d26b2b5321e 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -89,16 +89,33 @@ def _encode_nc4_variable(var): return var -def _get_datatype(var, nc_format='NETCDF4'): +def _check_encoding_dtype_is_vlen_string(dtype): + if dtype is not str: + raise AssertionError( # pragma: no cover + "unexpected dtype encoding %r. This shouldn't happen: please " + "file a bug report at github.com/pydata/xarray" % dtype) + + +def _get_datatype(var, nc_format='NETCDF4', raise_on_invalid_encoding=False): if nc_format == 'NETCDF4': datatype = _nc4_dtype(var) else: + if 'dtype' in var.encoding: + encoded_dtype = var.encoding['dtype'] + _check_encoding_dtype_is_vlen_string(encoded_dtype) + if raise_on_invalid_encoding: + raise ValueError( + 'encoding dtype=str for vlen strings is only supported ' + 'with format=\'NETCDF4\'.') datatype = var.dtype return datatype def _nc4_dtype(var): - if coding.strings.is_unicode_dtype(var.dtype): + if 'dtype' in var.encoding: + dtype = var.encoding.pop('dtype') + _check_encoding_dtype_is_vlen_string(dtype) + elif coding.strings.is_unicode_dtype(var.dtype): dtype = str elif var.dtype.kind in ['i', 'u', 'f', 'c', 'S']: dtype = var.dtype @@ -172,7 +189,7 @@ def _extract_nc4_variable_encoding(variable, raise_on_invalid=False, safe_to_drop = set(['source', 'original_shape']) valid_encodings = set(['zlib', 'complevel', 'fletcher32', 'contiguous', - 'chunksizes', 'shuffle', '_FillValue']) + 'chunksizes', 'shuffle', '_FillValue', 'dtype']) if lsd_okay: valid_encodings.add('least_significant_digit') if h5py_okay: @@ -344,6 +361,7 @@ def open_store_variable(self, name, var): # save source so __repr__ can detect if it's local or not encoding['source'] = self._filename encoding['original_shape'] = var.shape + encoding['dtype'] = var.dtype return Variable(dimensions, data, attributes, encoding) @@ -398,7 +416,8 @@ def encode_variable(self, variable): def prepare_variable(self, name, variable, check_encoding=False, unlimited_dims=None): - datatype = _get_datatype(variable, self.format) + datatype = _get_datatype(variable, self.format, + raise_on_invalid_encoding=check_encoding) attrs = variable.attrs.copy() fill_value = attrs.pop('_FillValue', None) diff --git a/xarray/coding/strings.py b/xarray/coding/strings.py index 08edeed4153..87b17d9175e 100644 --- a/xarray/coding/strings.py +++ b/xarray/coding/strings.py @@ -43,7 +43,10 @@ def encode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_encoding(variable) contains_unicode = is_unicode_dtype(data.dtype) - encode_as_char = 'dtype' in encoding and encoding['dtype'] == 'S1' + encode_as_char = encoding.get('dtype') == 'S1' + + if encode_as_char: + del encoding['dtype'] # no longer relevant if contains_unicode and (encode_as_char or not self.allows_unicode): if '_FillValue' in attrs: @@ -100,7 +103,7 @@ def encode(self, variable, name=None): variable = ensure_fixed_length_bytes(variable) dims, data, attrs, encoding = unpack_for_encoding(variable) - if data.dtype.kind == 'S': + if data.dtype.kind == 'S' and encoding.get('dtype') is not str: data = bytes_to_char(data) dims = dims + ('string%s' % data.shape[-1],) return Variable(dims, data, attrs, encoding) diff --git a/xarray/conventions.py b/xarray/conventions.py index 6171c353a0d..67dcb8d6d4e 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -79,7 +79,8 @@ def _var_as_tuple(var): def maybe_encode_nonstring_dtype(var, name=None): - if 'dtype' in var.encoding and var.encoding['dtype'] != 'S1': + if ('dtype' in var.encoding and + var.encoding['dtype'] not in ('S1', str)): dims, data, attrs, encoding = _var_as_tuple(var) dtype = np.dtype(encoding.pop('dtype')) if dtype != var.dtype: @@ -307,12 +308,7 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True, data = NativeEndiannessArray(data) original_dtype = data.dtype - if 'dtype' in encoding: - if original_dtype != encoding['dtype']: - warnings.warn("CF decoding is overwriting dtype on variable {!r}" - .format(name)) - else: - encoding['dtype'] = original_dtype + encoding.setdefault('dtype', original_dtype) if 'dtype' in attributes and attributes['dtype'] == 'bool': del attributes['dtype'] diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index b80cb18e2be..1e7a09fa55a 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -753,6 +753,7 @@ def test_encoding_kwarg(self): with self.roundtrip(ds, save_kwargs=kwargs) as actual: pass + def test_encoding_kwarg_dates(self): ds = Dataset({'t': pd.date_range('2000-01-01', periods=3)}) units = 'days since 1900-01-01' kwargs = dict(encoding={'t': {'units': units}}) @@ -760,6 +761,18 @@ def test_encoding_kwarg(self): self.assertEqual(actual.t.encoding['units'], units) assert_identical(actual, ds) + def test_encoding_kwarg_fixed_width_string(self): + # regression test for GH2149 + for strings in [ + [b'foo', b'bar', b'baz'], + [u'foo', u'bar', u'baz'], + ]: + ds = Dataset({'x': strings}) + kwargs = dict(encoding={'x': {'dtype': 'S1'}}) + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + self.assertEqual(actual['x'].encoding['dtype'], 'S1') + assert_identical(actual, ds) + def test_default_fill_value(self): # Test default encoding for float: ds = Dataset({'x': ('y', np.arange(10.0))}) @@ -879,8 +892,8 @@ def create_tmp_files(nfiles, suffix='.nc', allow_cleanup_failure=False): yield files -@requires_netCDF4 class BaseNetCDF4Test(CFEncodedDataTest): + """Tests for both netCDF4-python and h5netcdf.""" engine = 'netcdf4' @@ -942,6 +955,18 @@ def test_write_groups(self): with self.open(tmp_file, group='data/2') as actual2: assert_identical(data2, actual2) + def test_encoding_kwarg_vlen_string(self): + for input_strings in [ + [b'foo', b'bar', b'baz'], + [u'foo', u'bar', u'baz'], + ]: + original = Dataset({'x': input_strings}) + expected = Dataset({'x': [u'foo', u'bar', u'baz']}) + kwargs = dict(encoding={'x': {'dtype': str}}) + with self.roundtrip(original, save_kwargs=kwargs) as actual: + assert actual['x'].encoding['dtype'] is str + assert_identical(actual, expected) + def test_roundtrip_string_with_fill_value_vlen(self): values = np.array([u'ab', u'cdef', np.nan], dtype=object) expected = Dataset({'x': ('t', values)}) @@ -1054,6 +1079,23 @@ def test_compression_encoding(self): with self.roundtrip(expected) as actual: assert_equal(expected, actual) + def test_encoding_kwarg_compression(self): + ds = Dataset({'x': np.arange(10.0)}) + encoding = dict(dtype='f4', zlib=True, complevel=9, fletcher32=True, + chunksizes=(5,), shuffle=True) + kwargs = dict(encoding=dict(x=encoding)) + + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + assert_equal(actual, ds) + self.assertEqual(actual.x.encoding['dtype'], 'f4') + self.assertEqual(actual.x.encoding['zlib'], True) + self.assertEqual(actual.x.encoding['complevel'], 9) + self.assertEqual(actual.x.encoding['fletcher32'], True) + self.assertEqual(actual.x.encoding['chunksizes'], (5,)) + self.assertEqual(actual.x.encoding['shuffle'], True) + + self.assertEqual(ds.x.encoding, {}) + def test_encoding_chunksizes_unlimited(self): # regression test for GH1225 ds = Dataset({'x': [1, 2, 3], 'y': ('x', [2, 3, 4])}) @@ -1117,7 +1159,7 @@ def test_already_open_dataset(self): expected = Dataset({'x': ((), 42)}) assert_identical(expected, ds) - def test_variable_len_strings(self): + def test_read_variable_len_strings(self): with create_tmp_file() as tmp_file: values = np.array(['foo', 'bar', 'baz'], dtype=object) @@ -1410,6 +1452,10 @@ def test_group(self): open_kwargs={'group': group}) as actual: assert_identical(original, actual) + def test_encoding_kwarg_fixed_width_string(self): + # not relevant for zarr, since we don't use EncodedStringCoder + pass + # TODO: someone who understand caching figure out whether chaching # makes sense for Zarr backend @pytest.mark.xfail(reason="Zarr caching not implemented") @@ -1579,6 +1625,13 @@ def create_store(self): tmp_file, mode='w', format='NETCDF3_CLASSIC') as store: yield store + def test_encoding_kwarg_vlen_string(self): + original = Dataset({'x': [u'foo', u'bar', u'baz']}) + kwargs = dict(encoding={'x': {'dtype': str}}) + with raises_regex(ValueError, 'encoding dtype=str for vlen'): + with self.roundtrip(original, save_kwargs=kwargs): + pass + class NetCDF3ViaNetCDF4DataTestAutocloseTrue(NetCDF3ViaNetCDF4DataTest): autoclose = True diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index acc1c978579..5ed482ed2bd 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -272,7 +272,14 @@ def test_roundtrip_coordinates(self): 'CFEncodedInMemoryStore') def test_invalid_dataarray_names_raise(self): + # only relevant for on-disk file formats pass def test_encoding_kwarg(self): + # we haven't bothered to raise errors yet for unexpected encodings in + # this test dummy + pass + + def test_encoding_kwarg_fixed_width_string(self): + # CFEncodedInMemoryStore doesn't support explicit string encodings. pass From cf19528d6d2baf988ad34e024cae28361c9fd693 Mon Sep 17 00:00:00 2001 From: barronh Date: Fri, 1 Jun 2018 00:21:43 -0400 Subject: [PATCH 136/282] Added PNC backend to xarray (#1905) * Added PNC backend to xarray PNC is used for GEOS-Chem, CAMx, CMAQ and other atmospheric data formats that have their own file formats and meta-data conventions. It can provide a CF compliant netCDF-like interface. * Added whats-new documentation * Updating pnc_ to remove DunderArrayMixin dependency * Adding basic tests for pnc Right now, pnc is simply being tested as a reader for NetCDF3 files * Updating for flake8 compliance * flake does not like unused e * Updating pnc to PseudoNetCDF * Remove outer except * Updating pnc to PseudoNetCDF * Added open and updated init Based on shoyer review * Updated indexing and test fix Indexing supports #1899 * Added PseudoNetCDF to doc/io.rst * Changing test subtype * Changing test subtype removing pdb * pnc test case requires netcdf3only For now, pnc is only supporting the classic data model * adding backend_kwargs default as dict This ensures **mapping is possible. * Upgrading tests to CFEncodedDataTest Some tests are bypassed. PseudoNetCDF string treatment is not currently compatible with xarray. This will be addressed soon. * Not currently supporting autoclose I do not fully understand the usecase, so I have not implemented these tests. * Minor updates for flake8 * Explicit skipping Using pytest.mark.skip to skip unsupported tests * removing trailing whitespace from pytest skip * Adding pip support * Addressing comments * Bypassing pickle, mask/scale, and object These tests cause errors that do not affect desired backend performance. * Added uamiv test PseudoNetCDF reads other formats. This adds a test of uamiv to the standard test for a backend and skips mask/scale, object, and boolean tests * Adding support for autoclose ensure open must be called before accessing variable data * Adding bakcend_kwargs to all backends Most backends currently take no keywords, so an empty ditionary is appropriate. * Small tweaks to PNC backend * remove warning and update whats-new * Separating isntall and io pnc doc and updating whats new * fixing line length in test * Tests now use non-netcdf files * Removing unknown meta-data netcdf support. * flake8 cleanup * Using python 2 and 3 compat testing * Disabling mask_and_scale by default prevents inadvertent double scaling in PNC formats * consistent with 3.0.0 Updates in 3.0.1 will fix close in uamiv. * Updating readers and line length * Updating readers and line length * Updating readers and line length * Adding open_mfdataset test Testing by opening same file twice and stacking it. * Using conda version of PseudoNetCDF * Removing xfail for netcdf Mask and scale with PseudoNetCDF and NetCDF4 is not supported, but not prevented. * Moving pseudonetcdf to v0.15 * Updating what's new * Fixing open_dataarray CF options mask_and_scale is None (diagnosed by open_dataset) and decode_cf should be True --- ci/requirements-py36.yml | 1 + doc/installing.rst | 7 +- doc/io.rst | 23 ++- doc/whats-new.rst | 4 + xarray/backends/__init__.py | 2 + xarray/backends/api.py | 55 ++++++-- xarray/backends/pseudonetcdf_.py | 101 ++++++++++++++ xarray/tests/__init__.py | 1 + xarray/tests/data/example.ict | 31 +++++ xarray/tests/data/example.uamiv | Bin 0 -> 608 bytes xarray/tests/test_backends.py | 232 ++++++++++++++++++++++++++++++- 11 files changed, 440 insertions(+), 17 deletions(-) create mode 100644 xarray/backends/pseudonetcdf_.py create mode 100644 xarray/tests/data/example.ict create mode 100644 xarray/tests/data/example.uamiv diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index 0790f20764d..fd63fe26130 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -20,6 +20,7 @@ dependencies: - rasterio - bottleneck - zarr + - pseudonetcdf>=3.0.1 - pip: - coveralls - pytest-cov diff --git a/doc/installing.rst b/doc/installing.rst index bb42129deea..33f01b8c770 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -28,6 +28,9 @@ For netCDF and IO - `cftime `__: recommended if you want to encode/decode datetimes for non-standard calendars or dates before year 1678 or after year 2262. +- `PseudoNetCDF `__: recommended + for accessing CAMx, GEOS-Chem (bpch), NOAA ARL files, ICARTT files + (ffi1001) and many other. For accelerating xarray ~~~~~~~~~~~~~~~~~~~~~~~ @@ -65,9 +68,9 @@ with its recommended dependencies using the conda command line tool:: .. _conda: http://conda.io/ -We recommend using the community maintained `conda-forge `__ channel if you need difficult\-to\-build dependencies such as cartopy or pynio:: +We recommend using the community maintained `conda-forge `__ channel if you need difficult\-to\-build dependencies such as cartopy, pynio or PseudoNetCDF:: - $ conda install -c conda-forge xarray cartopy pynio + $ conda install -c conda-forge xarray cartopy pynio pseudonetcdf New releases may also appear in conda-forge before being updated in the default channel. diff --git a/doc/io.rst b/doc/io.rst index 668416e714d..e92ecd01cb4 100644 --- a/doc/io.rst +++ b/doc/io.rst @@ -650,7 +650,26 @@ We recommend installing PyNIO via conda:: .. _PyNIO: https://www.pyngl.ucar.edu/Nio.shtml -.. _combining multiple files: +.. _io.PseudoNetCDF: + +Formats supported by PseudoNetCDF +--------------------------------- + +xarray can also read CAMx, BPCH, ARL PACKED BIT, and many other file +formats supported by PseudoNetCDF_, if PseudoNetCDF is installed. +PseudoNetCDF can also provide Climate Forecasting Conventions to +CMAQ files. In addition, PseudoNetCDF can automatically register custom +readers that subclass PseudoNetCDF.PseudoNetCDFFile. PseudoNetCDF can +identify readers heuristically, or format can be specified via a key in +`backend_kwargs`. + +To use PseudoNetCDF to read such files, supply +``engine='pseudonetcdf'`` to :py:func:`~xarray.open_dataset`. + +Add ``backend_kwargs={'format': ''}`` where `` +options are listed on the PseudoNetCDF page. + +.. _PseuodoNetCDF: http://github.com/barronh/PseudoNetCDF Formats supported by Pandas @@ -662,6 +681,8 @@ exporting your objects to pandas and using its broad range of `IO tools`_. .. _IO tools: http://pandas.pydata.org/pandas-docs/stable/io.html +.. _combining multiple files: + Combining multiple files ------------------------ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index c4c8db243d4..bfa24340bcd 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -41,6 +41,10 @@ Enhancements dask<0.17.4. (related to :issue:`2203`) By `Keisuke Fujii `_. + - :py:meth:`~DataArray.cumsum` and :py:meth:`~DataArray.cumprod` now support aggregation over multiple dimensions at the same time. This is the default behavior when dimensions are not specified (previously this raised an error). diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index d85893afb0b..47a2011a3af 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -10,6 +10,7 @@ from .pynio_ import NioDataStore from .scipy_ import ScipyDataStore from .h5netcdf_ import H5NetCDFStore +from .pseudonetcdf_ import PseudoNetCDFDataStore from .zarr import ZarrStore __all__ = [ @@ -21,4 +22,5 @@ 'ScipyDataStore', 'H5NetCDFStore', 'ZarrStore', + 'PseudoNetCDFDataStore', ] diff --git a/xarray/backends/api.py b/xarray/backends/api.py index c3b2aa59fcd..753f8394a7b 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -152,9 +152,10 @@ def _finalize_store(write, store): def open_dataset(filename_or_obj, group=None, decode_cf=True, - mask_and_scale=True, decode_times=True, autoclose=False, + mask_and_scale=None, decode_times=True, autoclose=False, concat_characters=True, decode_coords=True, engine=None, - chunks=None, lock=None, cache=None, drop_variables=None): + chunks=None, lock=None, cache=None, drop_variables=None, + backend_kwargs=None): """Load and decode a dataset from a file or file-like object. Parameters @@ -178,7 +179,8 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, taken from variable attributes (if they exist). If the `_FillValue` or `missing_value` attribute contains multiple values a warning will be issued and all array values matching one of the multiple values will - be replaced by NA. + be replaced by NA. mask_and_scale defaults to True except for the + pseudonetcdf backend. decode_times : bool, optional If True, decode times encoded in the standard NetCDF datetime format into datetime objects. Otherwise, leave them encoded as numbers. @@ -194,7 +196,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, decode_coords : bool, optional If True, decode the 'coordinates' attribute to identify coordinates in the resulting dataset. - engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio'}, optional + engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', 'pseudonetcdf'}, optional Engine to use when reading files. If not provided, the default engine is chosen based on available dependencies, with a preference for 'netcdf4'. @@ -219,6 +221,10 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, A variable or list of variables to exclude from being parsed from the dataset. This may be useful to drop variables with problems or inconsistent values. + backend_kwargs: dictionary, optional + A dictionary of keyword arguments to pass on to the backend. This + may be useful when backend options would improve performance or + allow user control of dataset processing. Returns ------- @@ -229,6 +235,10 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, -------- open_mfdataset """ + + if mask_and_scale is None: + mask_and_scale = not engine == 'pseudonetcdf' + if not decode_cf: mask_and_scale = False decode_times = False @@ -238,6 +248,9 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, if cache is None: cache = chunks is None + if backend_kwargs is None: + backend_kwargs = {} + def maybe_decode_store(store, lock=False): ds = conventions.decode_cf( store, mask_and_scale=mask_and_scale, decode_times=decode_times, @@ -303,18 +316,26 @@ def maybe_decode_store(store, lock=False): if engine == 'netcdf4': store = backends.NetCDF4DataStore.open(filename_or_obj, group=group, - autoclose=autoclose) + autoclose=autoclose, + **backend_kwargs) elif engine == 'scipy': store = backends.ScipyDataStore(filename_or_obj, - autoclose=autoclose) + autoclose=autoclose, + **backend_kwargs) elif engine == 'pydap': - store = backends.PydapDataStore.open(filename_or_obj) + store = backends.PydapDataStore.open(filename_or_obj, + **backend_kwargs) elif engine == 'h5netcdf': store = backends.H5NetCDFStore(filename_or_obj, group=group, - autoclose=autoclose) + autoclose=autoclose, + **backend_kwargs) elif engine == 'pynio': store = backends.NioDataStore(filename_or_obj, - autoclose=autoclose) + autoclose=autoclose, + **backend_kwargs) + elif engine == 'pseudonetcdf': + store = backends.PseudoNetCDFDataStore.open( + filename_or_obj, autoclose=autoclose, **backend_kwargs) else: raise ValueError('unrecognized engine for open_dataset: %r' % engine) @@ -334,9 +355,10 @@ def maybe_decode_store(store, lock=False): def open_dataarray(filename_or_obj, group=None, decode_cf=True, - mask_and_scale=True, decode_times=True, autoclose=False, + mask_and_scale=None, decode_times=True, autoclose=False, concat_characters=True, decode_coords=True, engine=None, - chunks=None, lock=None, cache=None, drop_variables=None): + chunks=None, lock=None, cache=None, drop_variables=None, + backend_kwargs=None): """Open an DataArray from a netCDF file containing a single data variable. This is designed to read netCDF files with only one data variable. If @@ -363,7 +385,8 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, taken from variable attributes (if they exist). If the `_FillValue` or `missing_value` attribute contains multiple values a warning will be issued and all array values matching one of the multiple values will - be replaced by NA. + be replaced by NA. mask_and_scale defaults to True except for the + pseudonetcdf backend. decode_times : bool, optional If True, decode times encoded in the standard NetCDF datetime format into datetime objects. Otherwise, leave them encoded as numbers. @@ -403,6 +426,10 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, A variable or list of variables to exclude from being parsed from the dataset. This may be useful to drop variables with problems or inconsistent values. + backend_kwargs: dictionary, optional + A dictionary of keyword arguments to pass on to the backend. This + may be useful when backend options would improve performance or + allow user control of dataset processing. Notes ----- @@ -417,13 +444,15 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, -------- open_dataset """ + dataset = open_dataset(filename_or_obj, group=group, decode_cf=decode_cf, mask_and_scale=mask_and_scale, decode_times=decode_times, autoclose=autoclose, concat_characters=concat_characters, decode_coords=decode_coords, engine=engine, chunks=chunks, lock=lock, cache=cache, - drop_variables=drop_variables) + drop_variables=drop_variables, + backend_kwargs=backend_kwargs) if len(dataset.data_vars) != 1: raise ValueError('Given file dataset contains more than one data ' diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py new file mode 100644 index 00000000000..c481bf848b9 --- /dev/null +++ b/xarray/backends/pseudonetcdf_.py @@ -0,0 +1,101 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +import numpy as np + +from .. import Variable +from ..core.pycompat import OrderedDict +from ..core.utils import (FrozenOrderedDict, Frozen) +from ..core import indexing + +from .common import AbstractDataStore, DataStorePickleMixin, BackendArray + + +class PncArrayWrapper(BackendArray): + + def __init__(self, variable_name, datastore): + self.datastore = datastore + self.variable_name = variable_name + array = self.get_array() + self.shape = array.shape + self.dtype = np.dtype(array.dtype) + + def get_array(self): + self.datastore.assert_open() + return self.datastore.ds.variables[self.variable_name] + + def __getitem__(self, key): + key, np_inds = indexing.decompose_indexer( + key, self.shape, indexing.IndexingSupport.OUTER_1VECTOR) + + with self.datastore.ensure_open(autoclose=True): + array = self.get_array()[key.tuple] # index backend array + + if len(np_inds.tuple) > 0: + # index the loaded np.ndarray + array = indexing.NumpyIndexingAdapter(array)[np_inds] + return array + + +class PseudoNetCDFDataStore(AbstractDataStore, DataStorePickleMixin): + """Store for accessing datasets via PseudoNetCDF + """ + @classmethod + def open(cls, filename, format=None, writer=None, + autoclose=False, **format_kwds): + from PseudoNetCDF import pncopen + opener = functools.partial(pncopen, filename, **format_kwds) + ds = opener() + mode = format_kwds.get('mode', 'r') + return cls(ds, mode=mode, writer=writer, opener=opener, + autoclose=autoclose) + + def __init__(self, pnc_dataset, mode='r', writer=None, opener=None, + autoclose=False): + + if autoclose and opener is None: + raise ValueError('autoclose requires an opener') + + self._ds = pnc_dataset + self._autoclose = autoclose + self._isopen = True + self._opener = opener + self._mode = mode + super(PseudoNetCDFDataStore, self).__init__() + + def open_store_variable(self, name, var): + with self.ensure_open(autoclose=False): + data = indexing.LazilyOuterIndexedArray( + PncArrayWrapper(name, self) + ) + attrs = OrderedDict((k, getattr(var, k)) for k in var.ncattrs()) + return Variable(var.dimensions, data, attrs) + + def get_variables(self): + with self.ensure_open(autoclose=False): + return FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in self.ds.variables.items()) + + def get_attrs(self): + with self.ensure_open(autoclose=True): + return Frozen(dict([(k, getattr(self.ds, k)) + for k in self.ds.ncattrs()])) + + def get_dimensions(self): + with self.ensure_open(autoclose=True): + return Frozen(self.ds.dimensions) + + def get_encoding(self): + encoding = {} + encoding['unlimited_dims'] = set( + [k for k in self.ds.dimensions + if self.ds.dimensions[k].isunlimited()]) + return encoding + + def close(self): + if self._isopen: + self.ds.close() + self._isopen = False diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 3acd26235ce..e93d9a80145 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -68,6 +68,7 @@ def _importorskip(modname, minversion=None): has_netCDF4, requires_netCDF4 = _importorskip('netCDF4') has_h5netcdf, requires_h5netcdf = _importorskip('h5netcdf') has_pynio, requires_pynio = _importorskip('Nio') +has_pseudonetcdf, requires_pseudonetcdf = _importorskip('PseudoNetCDF') has_cftime, requires_cftime = _importorskip('cftime') has_dask, requires_dask = _importorskip('dask') has_bottleneck, requires_bottleneck = _importorskip('bottleneck') diff --git a/xarray/tests/data/example.ict b/xarray/tests/data/example.ict new file mode 100644 index 00000000000..bc04888fb80 --- /dev/null +++ b/xarray/tests/data/example.ict @@ -0,0 +1,31 @@ +27, 1001 +Henderson, Barron +U.S. EPA +Example file with artificial data +JUST_A_TEST +1, 1 +2018, 04, 27, 2018, 04, 27 +0 +Start_UTC +7 +1, 1, 1, 1, 1 +-9999, -9999, -9999, -9999, -9999 +lat, degrees_north +lon, degrees_east +elev, meters +TEST_ppbv, ppbv +TESTM_ppbv, ppbv +0 +8 +ULOD_FLAG: -7777 +ULOD_VALUE: N/A +LLOD_FLAG: -8888 +LLOD_VALUE: N/A, N/A, N/A, N/A, 0.025 +OTHER_COMMENTS: www-air.larc.nasa.gov/missions/etc/IcarttDataFormat.htm +REVISION: R0 +R0: No comments for this revision. +Start_UTC, lat, lon, elev, TEST_ppbv, TESTM_ppbv +43200, 41.00000, -71.00000, 5, 1.2345, 2.220 +46800, 42.00000, -72.00000, 15, 2.3456, -9999 +50400, 42.00000, -73.00000, 20, 3.4567, -7777 +50400, 42.00000, -74.00000, 25, 4.5678, -8888 \ No newline at end of file diff --git a/xarray/tests/data/example.uamiv b/xarray/tests/data/example.uamiv new file mode 100644 index 0000000000000000000000000000000000000000..fcedcd53097122839b5b94d1fabd2cb70d7c003e GIT binary patch literal 608 zcmb8rv1$TA5XSL2h+tviBU~jm38#rx0dEtcl_(OdQiP~rL`g_OlET6=WlBpQ#a5rf zN6G)YTSY{W4E%29cK2pS&4S2zd|YF)5KZjoR;gEOedvC#K<_&avzwN`8~gXTIF xI-B-6oH6M=RsVnVGX1`ok76FN>IIhAm^lN}xe)vVE=C)Vc*P7q_{H25(?3@UPTv3k literal 0 HcmV?d00001 diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 1e7a09fa55a..0e6151b2db5 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -32,7 +32,7 @@ assert_identical, has_dask, has_netCDF4, has_scipy, network, raises_regex, requires_dask, requires_h5netcdf, requires_netCDF4, requires_pathlib, requires_pydap, requires_pynio, requires_rasterio, requires_scipy, - requires_scipy_or_netCDF4, requires_zarr, + requires_scipy_or_netCDF4, requires_zarr, requires_pseudonetcdf, requires_cftime) from .test_dataset import create_test_data @@ -63,6 +63,13 @@ def open_example_dataset(name, *args, **kwargs): *args, **kwargs) +def open_example_mfdataset(names, *args, **kwargs): + return open_mfdataset( + [os.path.join(os.path.dirname(__file__), 'data', name) + for name in names], + *args, **kwargs) + + def create_masked_and_scaled_data(): x = np.array([np.nan, np.nan, 10, 10.1, 10.2], dtype=np.float32) encoding = {'_FillValue': -1, 'add_offset': 10, @@ -2483,6 +2490,229 @@ class PyNioTestAutocloseTrue(PyNioTest): autoclose = True +@requires_pseudonetcdf +class PseudoNetCDFFormatTest(TestCase): + autoclose = True + + def open(self, path, **kwargs): + return open_dataset(path, engine='pseudonetcdf', + autoclose=self.autoclose, + **kwargs) + + @contextlib.contextmanager + def roundtrip(self, data, save_kwargs={}, open_kwargs={}, + allow_cleanup_failure=False): + with create_tmp_file( + allow_cleanup_failure=allow_cleanup_failure) as path: + self.save(data, path, **save_kwargs) + with self.open(path, **open_kwargs) as ds: + yield ds + + def test_ict_format(self): + """ + Open a CAMx file and test data variables + """ + ictfile = open_example_dataset('example.ict', + engine='pseudonetcdf', + autoclose=False, + backend_kwargs={'format': 'ffi1001'}) + stdattr = { + 'fill_value': -9999.0, + 'missing_value': -9999, + 'scale': 1, + 'llod_flag': -8888, + 'llod_value': 'N/A', + 'ulod_flag': -7777, + 'ulod_value': 'N/A' + } + + def myatts(**attrs): + outattr = stdattr.copy() + outattr.update(attrs) + return outattr + + input = { + 'coords': {}, + 'attrs': { + 'fmt': '1001', 'n_header_lines': 27, + 'PI_NAME': 'Henderson, Barron', + 'ORGANIZATION_NAME': 'U.S. EPA', + 'SOURCE_DESCRIPTION': 'Example file with artificial data', + 'MISSION_NAME': 'JUST_A_TEST', + 'VOLUME_INFO': '1, 1', + 'SDATE': '2018, 04, 27', 'WDATE': '2018, 04, 27', + 'TIME_INTERVAL': '0', + 'INDEPENDENT_VARIABLE': 'Start_UTC', + 'ULOD_FLAG': '-7777', 'ULOD_VALUE': 'N/A', + 'LLOD_FLAG': '-8888', + 'LLOD_VALUE': ('N/A, N/A, N/A, N/A, 0.025'), + 'OTHER_COMMENTS': ('www-air.larc.nasa.gov/missions/etc/' + + 'IcarttDataFormat.htm'), + 'REVISION': 'R0', + 'R0': 'No comments for this revision.', + 'TFLAG': 'Start_UTC' + }, + 'dims': {'POINTS': 4}, + 'data_vars': { + 'Start_UTC': { + 'data': [43200.0, 46800.0, 50400.0, 50400.0], + 'dims': ('POINTS',), + 'attrs': myatts( + units='Start_UTC', + standard_name='Start_UTC', + ) + }, + 'lat': { + 'data': [41.0, 42.0, 42.0, 42.0], + 'dims': ('POINTS',), + 'attrs': myatts( + units='degrees_north', + standard_name='lat', + ) + }, + 'lon': { + 'data': [-71.0, -72.0, -73.0, -74.], + 'dims': ('POINTS',), + 'attrs': myatts( + units='degrees_east', + standard_name='lon', + ) + }, + 'elev': { + 'data': [5.0, 15.0, 20.0, 25.0], + 'dims': ('POINTS',), + 'attrs': myatts( + units='meters', + standard_name='elev', + ) + }, + 'TEST_ppbv': { + 'data': [1.2345, 2.3456, 3.4567, 4.5678], + 'dims': ('POINTS',), + 'attrs': myatts( + units='ppbv', + standard_name='TEST_ppbv', + ) + }, + 'TESTM_ppbv': { + 'data': [2.22, -9999.0, -7777.0, -8888.0], + 'dims': ('POINTS',), + 'attrs': myatts( + units='ppbv', + standard_name='TESTM_ppbv', + llod_value=0.025 + ) + } + } + } + chkfile = Dataset.from_dict(input) + assert_identical(ictfile, chkfile) + + def test_ict_format_write(self): + fmtkw = {'format': 'ffi1001'} + expected = open_example_dataset('example.ict', + engine='pseudonetcdf', + autoclose=False, + backend_kwargs=fmtkw) + with self.roundtrip(expected, save_kwargs=fmtkw, + open_kwargs={'backend_kwargs': fmtkw}) as actual: + assert_identical(expected, actual) + + def test_uamiv_format_read(self): + """ + Open a CAMx file and test data variables + """ + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=UserWarning, + message=('IOAPI_ISPH is assumed to be ' + + '6370000.; consistent with WRF')) + camxfile = open_example_dataset('example.uamiv', + engine='pseudonetcdf', + autoclose=True, + backend_kwargs={'format': 'uamiv'}) + data = np.arange(20, dtype='f').reshape(1, 1, 4, 5) + expected = xr.Variable(('TSTEP', 'LAY', 'ROW', 'COL'), data, + dict(units='ppm', long_name='O3'.ljust(16), + var_desc='O3'.ljust(80))) + actual = camxfile.variables['O3'] + assert_allclose(expected, actual) + + data = np.array(['2002-06-03'], 'datetime64[ns]') + expected = xr.Variable(('TSTEP',), data, + dict(bounds='time_bounds', + long_name=('synthesized time coordinate ' + + 'from SDATE, STIME, STEP ' + + 'global attributes'))) + actual = camxfile.variables['time'] + assert_allclose(expected, actual) + camxfile.close() + + def test_uamiv_format_mfread(self): + """ + Open a CAMx file and test data variables + """ + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=UserWarning, + message=('IOAPI_ISPH is assumed to be ' + + '6370000.; consistent with WRF')) + camxfile = open_example_mfdataset( + ['example.uamiv', + 'example.uamiv'], + engine='pseudonetcdf', + autoclose=True, + concat_dim='TSTEP', + backend_kwargs={'format': 'uamiv'}) + + data1 = np.arange(20, dtype='f').reshape(1, 1, 4, 5) + data = np.concatenate([data1] * 2, axis=0) + expected = xr.Variable(('TSTEP', 'LAY', 'ROW', 'COL'), data, + dict(units='ppm', long_name='O3'.ljust(16), + var_desc='O3'.ljust(80))) + actual = camxfile.variables['O3'] + assert_allclose(expected, actual) + + data1 = np.array(['2002-06-03'], 'datetime64[ns]') + data = np.concatenate([data1] * 2, axis=0) + expected = xr.Variable(('TSTEP',), data, + dict(bounds='time_bounds', + long_name=('synthesized time coordinate ' + + 'from SDATE, STIME, STEP ' + + 'global attributes'))) + actual = camxfile.variables['time'] + assert_allclose(expected, actual) + camxfile.close() + + def test_uamiv_format_write(self): + fmtkw = {'format': 'uamiv'} + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=UserWarning, + message=('IOAPI_ISPH is assumed to be ' + + '6370000.; consistent with WRF')) + expected = open_example_dataset('example.uamiv', + engine='pseudonetcdf', + autoclose=False, + backend_kwargs=fmtkw) + with self.roundtrip(expected, + save_kwargs=fmtkw, + open_kwargs={'backend_kwargs': fmtkw}) as actual: + assert_identical(expected, actual) + + def save(self, dataset, path, **save_kwargs): + import PseudoNetCDF as pnc + pncf = pnc.PseudoNetCDFFile() + pncf.dimensions = {k: pnc.PseudoNetCDFDimension(pncf, k, v) + for k, v in dataset.dims.items()} + pncf.variables = {k: pnc.PseudoNetCDFVariable(pncf, k, v.dtype.char, + v.dims, + values=v.data[...], + **v.attrs) + for k, v in dataset.variables.items()} + for pk, pv in dataset.attrs.items(): + setattr(pncf, pk, pv) + + pnc.pncwrite(pncf, path, **save_kwargs) + + @requires_rasterio @contextlib.contextmanager def create_tmp_geotiff(nx=4, ny=3, nz=3, From ad47ced88c1c99fd961617943e02613b67c9cea9 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 31 May 2018 22:17:25 -0700 Subject: [PATCH 137/282] Release v0.10.5 --- doc/whats-new.rst | 54 ++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index bfa24340bcd..4e4ed20a093 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -28,36 +28,25 @@ What's New .. _whats-new.0.10.5: -v0.10.5 (unreleased) --------------------- +v0.10.5 (31 May 2018) +--------------------- -Documentation -~~~~~~~~~~~~~ +The minor release includes a number of bug-fixes and backwards compatible +enhancements. Enhancements ~~~~~~~~~~~~ -- `:py:meth:`~DataArray.dot` and :py:func:`~dot` are partly supported with older - dask<0.17.4. (related to :issue:`2203`) - By `Keisuke Fujii `_. -- :py:meth:`~DataArray.cumsum` and :py:meth:`~DataArray.cumprod` now support - aggregation over multiple dimensions at the same time. This is the default - behavior when dimensions are not specified (previously this raised an error). - By `Stephan Hoyer `_ - -- Xarray now uses `Versioneer `__ - to manage its version strings. (:issue:`1300`). - By `Joe Hamman `_. - -- `:py:class:`Dataset`s align `:py:class:`DataArray`s to coords that are explicitly - passed into the constructor, where previously an error would be raised. +- The :py:class:`Dataset` constructor now aligns :py:class:`DataArray` + arguments in ``data_vars`` to indexes set explicitly in ``coords``, + where previously an error would be raised. (:issue:`674`) - By `Maximilian Roos `_. - :py:meth:`~DataArray.sel`, :py:meth:`~DataArray.isel` & :py:meth:`~DataArray.reindex`, (and their :py:class:`Dataset` counterparts) now support supplying a ``dict`` @@ -67,11 +56,24 @@ Enhancements not strings. By `Maximilian Roos `_. -- :py:meth:`~DataArray.rename` now supports supplying `kwargs`, as an +- :py:meth:`~DataArray.rename` now supports supplying ``**kwargs``, as an alternative to the existing approach of supplying a ``dict`` as the first argument. By `Maximilian Roos `_. +- :py:meth:`~DataArray.cumsum` and :py:meth:`~DataArray.cumprod` now support + aggregation over multiple dimensions at the same time. This is the default + behavior when dimensions are not specified (previously this raised an error). + By `Stephan Hoyer `_ + +- :py:meth:`~DataArray.dot` and :py:func:`~dot` are partly supported with older + dask<0.17.4. (related to :issue:`2203`) + By `Keisuke Fujii `_. + +- Xarray now uses `Versioneer `__ + to manage its version strings. (:issue:`1300`). + By `Joe Hamman `_. + Bug fixes ~~~~~~~~~ @@ -114,13 +116,13 @@ Bug fixes (:issue:`2153`). By `Stephan Hoyer `_ -- Fix Dataset.to_netcdf() cannot create group with engine="h5netcdf" +- Fix ``Dataset.to_netcdf()`` cannot create group with ``engine="h5netcdf"`` (:issue:`2177`). By `Stephan Hoyer `_ .. _whats-new.0.10.4: -v0.10.4 (May 16, 2018) +v0.10.4 (16 May 2018) ---------------------- The minor release includes a number of bug-fixes and backwards compatible @@ -208,7 +210,7 @@ Bug fixes .. _whats-new.0.10.3: -v0.10.3 (April 13, 2018) +v0.10.3 (13 April 2018) ------------------------ The minor release includes a number of bug-fixes and backwards compatible enhancements. From a3cf251cc04ed5731912171a6c2b63f8927e610e Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 31 May 2018 22:20:52 -0700 Subject: [PATCH 138/282] Add whats-new for v0.10.6 --- doc/whats-new.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4e4ed20a093..730ba20b850 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -26,6 +26,20 @@ What's New - `Tips on porting to Python 3 `__ +.. _whats-new.0.10.6: + +v0.10.6 (unreleased) +-------------------- + +Documentation +~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + .. _whats-new.0.10.5: v0.10.5 (31 May 2018) From 1c9b4b2e556b81f1e668ae7aa3aaea8aa91b7983 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 31 May 2018 22:45:58 -0700 Subject: [PATCH 139/282] Fix versioneer, release v0.10.6 --- doc/whats-new.rst | 17 +---------------- setup.cfg | 2 +- xarray/_version.py | 2 +- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 730ba20b850..ad88289e43f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,24 +25,9 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ - .. _whats-new.0.10.6: -v0.10.6 (unreleased) --------------------- - -Documentation -~~~~~~~~~~~~~ - -Enhancements -~~~~~~~~~~~~ - -Bug fixes -~~~~~~~~~ - -.. _whats-new.0.10.5: - -v0.10.5 (31 May 2018) +v0.10.6 (31 May 2018) --------------------- The minor release includes a number of bug-fixes and backwards compatible diff --git a/setup.cfg b/setup.cfg index 4dd1bffe043..17f24b3f1ce 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,7 @@ VCS = git style = pep440 versionfile_source = xarray/_version.py versionfile_build = xarray/_version.py -tag_prefix = +tag_prefix = v parentdir_prefix = xarray- [aliases] diff --git a/xarray/_version.py b/xarray/_version.py index 2fa32b69798..df4ee95ade4 100644 --- a/xarray/_version.py +++ b/xarray/_version.py @@ -41,7 +41,7 @@ def get_config(): cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "pep440" - cfg.tag_prefix = "" + cfg.tag_prefix = "v" cfg.parentdir_prefix = "xarray-" cfg.versionfile_source = "xarray/_version.py" cfg.verbose = False From 16ef0cf508d99b4154f41004992e648eb6cd6eb9 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 31 May 2018 22:51:57 -0700 Subject: [PATCH 140/282] Add whats-new for v0.10.7 --- doc/whats-new.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index ad88289e43f..ea1c2114a41 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,20 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ +.. _whats-new.0.10.7: + +v0.10.7 (unreleased) +-------------------- + +Documentation +~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + .. _whats-new.0.10.6: v0.10.6 (31 May 2018) From 1c37d9ce526fecb9fdab2c82b3f46be06f55a128 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 1 Jun 2018 12:15:49 -0400 Subject: [PATCH 141/282] Remove height=12in from facetgrid example plots. (#2210) --- doc/plotting.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index 28fbe7062a6..b10f0e7fc64 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -436,7 +436,7 @@ arguments to the xarray plotting methods/functions. This returns a .. ipython:: python - @savefig plot_facet_dataarray.png height=12in + @savefig plot_facet_dataarray.png g_simple = t.plot(x='lon', y='lat', col='time', col_wrap=3) 4 dimensional @@ -454,7 +454,7 @@ one were much hotter. # This is a 4d array t4d.coords - @savefig plot_facet_4d.png height=12in + @savefig plot_facet_4d.png t4d.plot(x='lon', y='lat', col='time', row='fourth_dim') Other features @@ -468,7 +468,7 @@ Faceted plotting supports other arguments common to xarray 2d plots. hasoutliers[0, 0, 0] = -100 hasoutliers[-1, -1, -1] = 400 - @savefig plot_facet_robust.png height=12in + @savefig plot_facet_robust.png g = hasoutliers.plot.pcolormesh('lon', 'lat', col='time', col_wrap=3, robust=True, cmap='viridis') @@ -509,7 +509,7 @@ they have been plotted. bottomright = g.axes[-1, -1] bottomright.annotate('bottom right', (240, 40)) - @savefig plot_facet_iterator.png height=12in + @savefig plot_facet_iterator.png plt.show() TODO: add an example of using the ``map`` method to plot dataset variables From 1e6984b247a08c5c98baa808dcc7552eb1c372a0 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 1 Jun 2018 20:10:25 -0400 Subject: [PATCH 142/282] Plot labels use CF convention information if available. (#2151) * Plot labels use CF convention information if available. Uses attrs long_name/standard_name, units if available. * Follow review feedback. * More informative docs. * Minor edit. --- doc/plotting.rst | 15 +++++- doc/whats-new.rst | 2 + xarray/plot/facetgrid.py | 27 +++++++---- xarray/plot/plot.py | 26 ++++++----- xarray/plot/utils.py | 22 +++++++++ xarray/tests/test_plot.py | 96 +++++++++++++++++++++++++++++---------- 6 files changed, 142 insertions(+), 46 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index b10f0e7fc64..fa364d4838e 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -66,6 +66,13 @@ For these examples we'll use the North American air temperature dataset. # Convert to celsius air = airtemps.air - 273.15 + # copy attributes to get nice figure labels and change Kelvin to Celsius + air.attrs = airtemps.air.attrs + air.attrs['units'] = 'deg C' + +.. note:: + Until :issue:`1614` is solved, you might need to copy over the metadata in ``attrs`` to get informative figure labels (as was done above). + One Dimension ------------- @@ -73,7 +80,7 @@ One Dimension Simple Example ~~~~~~~~~~~~~~ -xarray uses the coordinate name to label the x axis. +The simplest way to make a plot is to call the :py:func:`xarray.DataArray.plot()` method. .. ipython:: python @@ -82,6 +89,12 @@ xarray uses the coordinate name to label the x axis. @savefig plotting_1d_simple.png width=4in air1d.plot() +xarray uses the coordinate name along with metadata ``attrs.long_name``, ``attrs.standard_name``, ``DataArray.name`` and ``attrs.units`` (if available) to label the axes. The names ``long_name``, ``standard_name`` and ``units`` are copied from the `CF-conventions spec `_. When choosing names, the order of precedence is ``long_name``, ``standard_name`` and finally ``DataArray.name``. The y-axis label in the above plot was constructed from the ``long_name`` and ``units`` attributes of ``air1d``. + +.. ipython:: python + + air1d.attrs + Additional Arguments ~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/whats-new.rst b/doc/whats-new.rst index ea1c2114a41..749726ee190 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -35,6 +35,8 @@ Documentation Enhancements ~~~~~~~~~~~~ +- Plot labels now make use of metadata that follow CF conventions. + By `Deepak Cherian `_ and `Ryan Abernathey `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index 5abae214c9f..361b47262c9 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -9,7 +9,8 @@ from ..core.formatting import format_item from ..core.pycompat import getargspec from .utils import ( - _determine_cmap_params, _infer_xy_labels, import_matplotlib_pyplot) + _determine_cmap_params, _infer_xy_labels, import_matplotlib_pyplot, + label_from_attrs) # Overrides axes.labelsize, xtick.major.size, ytick.major.size # from mpl.rcParams @@ -282,8 +283,8 @@ def add_colorbar(self, **kwargs): kwargs = kwargs.copy() if self._cmap_extend is not None: kwargs.setdefault('extend', self._cmap_extend) - if getattr(self.data, 'name', None) is not None: - kwargs.setdefault('label', self.data.name) + if 'label' not in kwargs: + kwargs.setdefault('label', label_from_attrs(self.data)) self.cbar = self.fig.colorbar(self._mappables[-1], ax=list(self.axes.flat), **kwargs) @@ -292,17 +293,25 @@ def add_colorbar(self, **kwargs): def set_axis_labels(self, x_var=None, y_var=None): """Set axis labels on the left column and bottom row of the grid.""" if x_var is not None: - self._x_var = x_var - self.set_xlabels(x_var) + if x_var in self.data.coords: + self._x_var = x_var + self.set_xlabels(label_from_attrs(self.data[x_var])) + else: + # x_var is a string + self.set_xlabels(x_var) + if y_var is not None: - self._y_var = y_var - self.set_ylabels(y_var) + if y_var in self.data.coords: + self._y_var = y_var + self.set_ylabels(label_from_attrs(self.data[y_var])) + else: + self.set_ylabels(y_var) return self def set_xlabels(self, label=None, **kwargs): """Label the x axis on the bottom row of the grid.""" if label is None: - label = self._x_var + label = label_from_attrs(self.data[self._x_var]) for ax in self._bottom_axes: ax.set_xlabel(label, **kwargs) return self @@ -310,7 +319,7 @@ def set_xlabels(self, label=None, **kwargs): def set_ylabels(self, label=None, **kwargs): """Label the y axis on the left column of the grid.""" if label is None: - label = self._y_var + label = label_from_attrs(self.data[self._y_var]) for ax in self._left_axes: ax.set_ylabel(label, **kwargs) return self diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index ee1df611d3b..f49ec5c52d9 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -20,7 +20,7 @@ from .facetgrid import FacetGrid from .utils import ( ROBUST_PERCENTILE, _determine_cmap_params, _infer_xy_labels, get_axis, - import_matplotlib_pyplot) + import_matplotlib_pyplot, label_from_attrs) def _valid_numpy_subdtype(x, numpy_types): @@ -240,14 +240,10 @@ def line(darray, *args, **kwargs): if (x is None and y is None) or x == dim: xplt = darray.coords[dim] yplt = darray - xlabel = dim - ylabel = darray.name else: yplt = darray.coords[dim] xplt = darray - xlabel = darray.name - ylabel = dim else: if x is None and y is None and hue is None: @@ -265,6 +261,12 @@ def line(darray, *args, **kwargs): xplt = darray.transpose(ylabel, huelabel) yplt = darray.coords[ylabel] + huecoords = darray[huelabel] + huelabel = label_from_attrs(huecoords) + + xlabel = label_from_attrs(xplt) + ylabel = label_from_attrs(yplt) + _ensure_plottable(xplt) primitive = ax.plot(xplt, yplt, *args, **kwargs) @@ -279,7 +281,7 @@ def line(darray, *args, **kwargs): if darray.ndim == 2 and add_legend: ax.legend(handles=primitive, - labels=list(darray.coords[huelabel].values), + labels=list(huecoords.values), title=huelabel) # Rotate dates on xlabels @@ -333,8 +335,8 @@ def hist(darray, figsize=None, size=None, aspect=None, ax=None, **kwargs): ax.set_ylabel('Count') - if darray.name is not None: - ax.set_title('Histogram of {0}'.format(darray.name)) + ax.set_title('Histogram') + ax.set_xlabel(label_from_attrs(darray)) return primitive @@ -652,8 +654,8 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, # Label the plot with metadata if add_labels: - ax.set_xlabel(xlab) - ax.set_ylabel(ylab) + ax.set_xlabel(label_from_attrs(darray[xlab])) + ax.set_ylabel(label_from_attrs(darray[ylab])) ax.set_title(darray._title_for_slice()) if add_colorbar: @@ -664,8 +666,8 @@ def newplotfunc(darray, x=None, y=None, figsize=None, size=None, else: cbar_kwargs.setdefault('cax', cbar_ax) cbar = plt.colorbar(primitive, **cbar_kwargs) - if darray.name and add_labels and 'label' not in cbar_kwargs: - cbar.set_label(darray.name, rotation=90) + if add_labels and 'label' not in cbar_kwargs: + cbar.set_label(label_from_attrs(darray), rotation=90) elif cbar_ax is not None or cbar_kwargs is not None: # inform the user about keywords which aren't used raise ValueError("cbar_ax and cbar_kwargs can't be used with " diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 7ba48819518..6846c553b8b 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd import pkg_resources +import textwrap from ..core.pycompat import basestring from ..core.utils import is_scalar @@ -354,3 +355,24 @@ def get_axis(figsize, size, aspect, ax): ax = plt.gca() return ax + + +def label_from_attrs(da): + ''' Makes informative labels if variable metadata (attrs) follows + CF conventions. ''' + + if da.attrs.get('long_name'): + name = da.attrs['long_name'] + elif da.attrs.get('standard_name'): + name = da.attrs['standard_name'] + elif da.name is not None: + name = da.name + else: + name = '' + + if da.attrs.get('units'): + units = ' [{}]'.format(da.attrs['units']) + else: + units = '' + + return '\n'.join(textwrap.wrap(name + units, 30)) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 70ed1156643..db1fb2fd081 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -13,7 +13,7 @@ from xarray.plot.plot import _infer_interval_breaks from xarray.plot.utils import ( _build_discrete_cmap, _color_palette, _determine_cmap_params, - import_seaborn) + import_seaborn, label_from_attrs) from . import ( TestCase, assert_array_equal, assert_equal, raises_regex, @@ -89,6 +89,28 @@ class TestPlot(PlotTestCase): def setUp(self): self.darray = DataArray(easy_array((2, 3, 4))) + def test_label_from_attrs(self): + da = self.darray.copy() + assert '' == label_from_attrs(da) + + da.name = 'a' + da.attrs['units'] = 'a_units' + da.attrs['long_name'] = 'a_long_name' + da.attrs['standard_name'] = 'a_standard_name' + assert 'a_long_name [a_units]' == label_from_attrs(da) + + da.attrs.pop('long_name') + assert 'a_standard_name [a_units]' == label_from_attrs(da) + da.attrs.pop('units') + assert 'a_standard_name' == label_from_attrs(da) + + da.attrs['units'] = 'a_units' + da.attrs.pop('standard_name') + assert 'a [a_units]' == label_from_attrs(da) + + da.attrs.pop('units') + assert 'a' == label_from_attrs(da) + def test1d(self): self.darray[:, 0, 0].plot() @@ -303,10 +325,11 @@ def setUp(self): d = [0, 1.1, 0, 2] self.darray = DataArray( d, coords={'period': range(len(d))}, dims='period') + self.darray.period.attrs['units'] = 's' def test_xlabel_is_index_name(self): self.darray.plot() - assert 'period' == plt.gca().get_xlabel() + assert 'period [s]' == plt.gca().get_xlabel() def test_no_label_name_on_x_axis(self): self.darray.plot(y='period') @@ -318,13 +341,15 @@ def test_no_label_name_on_y_axis(self): def test_ylabel_is_data_name(self): self.darray.name = 'temperature' + self.darray.attrs['units'] = 'degrees_Celsius' self.darray.plot() - assert self.darray.name == plt.gca().get_ylabel() + assert 'temperature [degrees_Celsius]' == plt.gca().get_ylabel() def test_xlabel_is_data_name(self): self.darray.name = 'temperature' + self.darray.attrs['units'] = 'degrees_Celsius' self.darray.plot(y='period') - self.assertEqual(self.darray.name, plt.gca().get_xlabel()) + assert 'temperature [degrees_Celsius]' == plt.gca().get_xlabel() def test_format_string(self): self.darray.plot.line('ro') @@ -374,19 +399,20 @@ def setUp(self): def test_3d_array(self): self.darray.plot.hist() - def test_title_no_name(self): - self.darray.plot.hist() - assert '' == plt.gca().get_title() - - def test_title_uses_name(self): + def test_xlabel_uses_name(self): self.darray.name = 'testpoints' + self.darray.attrs['units'] = 'testunits' self.darray.plot.hist() - assert self.darray.name in plt.gca().get_title() + assert 'testpoints [testunits]' == plt.gca().get_xlabel() def test_ylabel_is_count(self): self.darray.plot.hist() assert 'Count' == plt.gca().get_ylabel() + def test_title_is_histogram(self): + self.darray.plot.hist() + assert 'Histogram' == plt.gca().get_title() + def test_can_pass_in_kwargs(self): nbins = 5 self.darray.plot.hist(bins=nbins) @@ -654,7 +680,10 @@ class Common2dMixin: """ def setUp(self): - da = DataArray(easy_array((10, 15), start=-1), dims=['y', 'x']) + da = DataArray(easy_array((10, 15), start=-1), + dims=['y', 'x'], + coords={'y': np.arange(10), + 'x': np.arange(15)}) # add 2d coords ds = da.to_dataset(name='testvar') x, y = np.meshgrid(da.x.values, da.y.values) @@ -663,12 +692,21 @@ def setUp(self): ds.set_coords(['x2d', 'y2d'], inplace=True) # set darray and plot method self.darray = ds.testvar + + # Add CF-compliant metadata + self.darray.attrs['long_name'] = 'a_long_name' + self.darray.attrs['units'] = 'a_units' + self.darray.x.attrs['long_name'] = 'x_long_name' + self.darray.x.attrs['units'] = 'x_units' + self.darray.y.attrs['long_name'] = 'y_long_name' + self.darray.y.attrs['units'] = 'y_units' + self.plotmethod = getattr(self.darray.plot, self.plotfunc.__name__) def test_label_names(self): self.plotmethod() - assert 'x' == plt.gca().get_xlabel() - assert 'y' == plt.gca().get_ylabel() + assert 'x_long_name [x_units]' == plt.gca().get_xlabel() + assert 'y_long_name [y_units]' == plt.gca().get_ylabel() def test_1d_raises_valueerror(self): with raises_regex(ValueError, r'DataArray must be 2d'): @@ -761,19 +799,19 @@ def test_diverging_color_limits(self): def test_xy_strings(self): self.plotmethod('y', 'x') ax = plt.gca() - assert 'y' == ax.get_xlabel() - assert 'x' == ax.get_ylabel() + assert 'y_long_name [y_units]' == ax.get_xlabel() + assert 'x_long_name [x_units]' == ax.get_ylabel() def test_positional_coord_string(self): self.plotmethod(y='x') ax = plt.gca() - assert 'x' == ax.get_ylabel() - assert 'y' == ax.get_xlabel() + assert 'x_long_name [x_units]' == ax.get_ylabel() + assert 'y_long_name [y_units]' == ax.get_xlabel() self.plotmethod(x='x') ax = plt.gca() - assert 'x' == ax.get_xlabel() - assert 'y' == ax.get_ylabel() + assert 'x_long_name [x_units]' == ax.get_xlabel() + assert 'y_long_name [y_units]' == ax.get_ylabel() def test_bad_x_string_exception(self): with raises_regex(ValueError, 'x and y must be coordinate variables'): @@ -797,7 +835,7 @@ def test_non_linked_coords(self): # Normal case, without transpose self.plotfunc(self.darray, x='x', y='newy') ax = plt.gca() - assert 'x' == ax.get_xlabel() + assert 'x_long_name [x_units]' == ax.get_xlabel() assert 'newy' == ax.get_ylabel() # ax limits might change between plotfuncs # simply ensure that these high coords were passed over @@ -812,7 +850,7 @@ def test_non_linked_coords_transpose(self): self.plotfunc(self.darray, x='newy', y='x') ax = plt.gca() assert 'newy' == ax.get_xlabel() - assert 'x' == ax.get_ylabel() + assert 'x_long_name [x_units]' == ax.get_ylabel() # ax limits might change between plotfuncs # simply ensure that these high coords were passed over assert np.min(ax.get_xlim()) > 100. @@ -826,19 +864,29 @@ def test_default_title(self): assert 'c = 1, d = foo' == title or 'd = foo, c = 1' == title def test_colorbar_default_label(self): - self.darray.name = 'testvar' self.plotmethod(add_colorbar=True) - assert self.darray.name in text_in_fig() + assert ('a_long_name [a_units]' in text_in_fig()) def test_no_labels(self): self.darray.name = 'testvar' + self.darray.attrs['units'] = 'test_units' self.plotmethod(add_labels=False) alltxt = text_in_fig() - for string in ['x', 'y', 'testvar']: + for string in ['x_long_name [x_units]', + 'y_long_name [y_units]', + 'testvar [test_units]']: assert string not in alltxt def test_colorbar_kwargs(self): # replace label + self.darray.attrs.pop('long_name') + self.darray.attrs['units'] = 'test_units' + # check default colorbar label + self.plotmethod(add_colorbar=True) + alltxt = text_in_fig() + assert 'testvar [test_units]' in alltxt + self.darray.attrs.pop('units') + self.darray.name = 'testvar' self.plotmethod(add_colorbar=True, cbar_kwargs={'label': 'MyLabel'}) alltxt = text_in_fig() From 69c9c45bf7a9d572200c4649605a5875e96b650c Mon Sep 17 00:00:00 2001 From: crusaderky Date: Sat, 2 Jun 2018 13:15:32 +0100 Subject: [PATCH 143/282] Trivial docs fix (#2212) --- doc/io.rst | 2 +- doc/whats-new.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/io.rst b/doc/io.rst index e92ecd01cb4..7f7e7a2a66a 100644 --- a/doc/io.rst +++ b/doc/io.rst @@ -669,7 +669,7 @@ To use PseudoNetCDF to read such files, supply Add ``backend_kwargs={'format': ''}`` where `` options are listed on the PseudoNetCDF page. -.. _PseuodoNetCDF: http://github.com/barronh/PseudoNetCDF +.. _PseudoNetCDF: http://github.com/barronh/PseudoNetCDF Formats supported by Pandas diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 749726ee190..5e5da295186 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -81,7 +81,7 @@ Enhancements behavior when dimensions are not specified (previously this raised an error). By `Stephan Hoyer `_ -- :py:meth:`~DataArray.dot` and :py:func:`~dot` are partly supported with older +- :py:meth:`DataArray.dot` and :py:func:`dot` are partly supported with older dask<0.17.4. (related to :issue:`2203`) By `Keisuke Fujii `_. From bc52f8aa64833d8c97f9ef5253b6a78c7033f521 Mon Sep 17 00:00:00 2001 From: Yohai Bar Sinai <6164157+yohai@users.noreply.github.com> Date: Mon, 4 Jun 2018 11:54:44 -0400 Subject: [PATCH 144/282] ENH: added FacetGrid functionality to line plots (#2107) * ENH: added FacetGrid functionality to line plots a) plot.line can now accept also 'row' and 'col' keywords. b) If 'hue' is passed as a keyword to DataArray.plot() it generates a line plot FacetGrid. c) Line plots are automatically generated if the number of dimensions after faceting along row and/or col is one. * minor formatting issues * minor formatting issues * fix kwargs bug * added tests and refactoring line_legend * add documentation * added tests * minor formatting * Fix merge. All tests pass now. --- doc/plotting.rst | 15 ++- xarray/plot/facetgrid.py | 67 +++++++++++++- xarray/plot/plot.py | 187 ++++++++++++++++++++++++-------------- xarray/tests/test_plot.py | 66 ++++++++++++++ 4 files changed, 263 insertions(+), 72 deletions(-) diff --git a/doc/plotting.rst b/doc/plotting.rst index fa364d4838e..54fa2f57ac8 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -208,7 +208,11 @@ It is required to explicitly specify either 2. ``hue``: the dimension you want to represent by multiple lines. Thus, we could have made the previous plot by specifying ``hue='lat'`` instead of ``x='time'``. -If required, the automatic legend can be turned off using ``add_legend=False``. +If required, the automatic legend can be turned off using ``add_legend=False``. Alternatively, +``hue`` can be passed directly to :py:func:`xarray.plot` as `air.isel(lon=10, lat=[19,21,22]).plot(hue='lat')`. + + + Dimension along y-axis ~~~~~~~~~~~~~~~~~~~~~~ @@ -218,7 +222,7 @@ It is also possible to make line plots such that the data are on the x-axis and .. ipython:: python @savefig plotting_example_xy_kwarg.png - air.isel(time=10, lon=[10, 11]).plot.line(y='lat', hue='lon') + air.isel(time=10, lon=[10, 11]).plot(y='lat', hue='lon') Changing Axes Direction ----------------------- @@ -452,6 +456,13 @@ arguments to the xarray plotting methods/functions. This returns a @savefig plot_facet_dataarray.png g_simple = t.plot(x='lon', y='lat', col='time', col_wrap=3) +Faceting also works for line plots. + +.. ipython:: python + + @savefig plot_facet_dataarray_line.png + g_simple_line = t.isel(lat=slice(0,None,4)).plot(x='lon', hue='lat', col='time', col_wrap=3) + 4 dimensional ~~~~~~~~~~~~~ diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index 361b47262c9..771f0879408 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -5,7 +5,6 @@ import warnings import numpy as np - from ..core.formatting import format_item from ..core.pycompat import getargspec from .utils import ( @@ -267,6 +266,44 @@ def map_dataarray(self, func, x, y, **kwargs): return self + def map_dataarray_line(self, x=None, y=None, hue=None, **kwargs): + """ + Apply a line plot to a 2d facet subset of the data. + + Parameters + ---------- + x, y, hue: string + dimension names for the axes and hues of each facet + + Returns + ------- + self : FacetGrid object + + """ + from .plot import line, _infer_line_data + + add_legend = kwargs.pop('add_legend', True) + kwargs['add_legend'] = False + + for d, ax in zip(self.name_dicts.flat, self.axes.flat): + # None is the sentinel value + if d is not None: + subset = self.data.loc[d] + mappable = line(subset, x=x, y=y, hue=hue, + ax=ax, _labels=False, + **kwargs) + self._mappables.append(mappable) + _, _, _, xlabel, ylabel, huelabel = _infer_line_data( + darray=self.data.loc[self.name_dicts.flat[0]], + x=x, y=y, hue=hue) + + self._finalize_grid(xlabel, ylabel) + + if add_legend and huelabel: + self.add_line_legend(huelabel) + + return self + def _finalize_grid(self, *axlabels): """Finalize the annotations and layout.""" self.set_axis_labels(*axlabels) @@ -277,6 +314,34 @@ def _finalize_grid(self, *axlabels): if namedict is None: ax.set_visible(False) + def add_line_legend(self, huelabel): + figlegend = self.fig.legend( + handles=self._mappables[-1], + labels=list(self.data.coords[huelabel].values), + title=huelabel, + loc="center right") + + # Draw the plot to set the bounding boxes correctly + self.fig.draw(self.fig.canvas.get_renderer()) + + # Calculate and set the new width of the figure so the legend fits + legend_width = figlegend.get_window_extent().width / self.fig.dpi + figure_width = self.fig.get_figwidth() + self.fig.set_figwidth(figure_width + legend_width) + + # Draw the plot again to get the new transformations + self.fig.draw(self.fig.canvas.get_renderer()) + + # Now calculate how much space we need on the right side + legend_width = figlegend.get_window_extent().width / self.fig.dpi + space_needed = legend_width / (figure_width + legend_width) + 0.02 + # margin = .01 + # _space_needed = margin + space_needed + right = 1 - space_needed + + # Place the subplot axes to give space for the legend + self.fig.subplots_adjust(right=right) + def add_colorbar(self, **kwargs): """Draw a colorbar """ diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index f49ec5c52d9..6322fc09d92 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -83,8 +83,32 @@ def _easy_facetgrid(darray, plotfunc, x, y, row=None, col=None, return g.map_dataarray(plotfunc, x, y, **kwargs) -def plot(darray, row=None, col=None, col_wrap=None, ax=None, rtol=0.01, - subplot_kws=None, **kwargs): +def _line_facetgrid(darray, row=None, col=None, hue=None, + col_wrap=None, sharex=True, sharey=True, aspect=None, + size=None, subplot_kws=None, **kwargs): + """ + Convenience method to call xarray.plot.FacetGrid for line plots + kwargs are the arguments to pyplot.plot() + """ + ax = kwargs.pop('ax', None) + figsize = kwargs.pop('figsize', None) + if ax is not None: + raise ValueError("Can't use axes when making faceted plots.") + if aspect is None: + aspect = 1 + if size is None: + size = 3 + elif figsize is not None: + raise ValueError('cannot provide both `figsize` and `size` arguments') + + g = FacetGrid(data=darray, col=col, row=row, col_wrap=col_wrap, + sharex=sharex, sharey=sharey, figsize=figsize, + aspect=aspect, size=size, subplot_kws=subplot_kws) + return g.map_dataarray_line(hue=hue, **kwargs) + + +def plot(darray, row=None, col=None, col_wrap=None, ax=None, hue=None, + rtol=0.01, subplot_kws=None, **kwargs): """ Default plot of DataArray using matplotlib.pyplot. @@ -106,6 +130,8 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, rtol=0.01, If passed, make row faceted plots on this dimension name col : string, optional If passed, make column faceted plots on this dimension name + hue : string, optional + If passed, make faceted line plots with hue on this dimension name col_wrap : integer, optional Use together with ``col`` to wrap faceted plots ax : matplotlib axes, optional @@ -129,26 +155,28 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, rtol=0.01, plot_dims = set(darray.dims) plot_dims.discard(row) plot_dims.discard(col) + plot_dims.discard(hue) ndims = len(plot_dims) - error_msg = ('Only 2d plots are supported for facets in xarray. ' + error_msg = ('Only 1d and 2d plots are supported for facets in xarray. ' 'See the package `Seaborn` for more options.') - if ndims == 1: + if ndims in [1, 2]: if row or col: - raise ValueError(error_msg) - plotfunc = line - elif ndims == 2: - # Only 2d can FacetGrid - kwargs['row'] = row - kwargs['col'] = col - kwargs['col_wrap'] = col_wrap - kwargs['subplot_kws'] = subplot_kws - - plotfunc = pcolormesh + kwargs['row'] = row + kwargs['col'] = col + kwargs['col_wrap'] = col_wrap + kwargs['subplot_kws'] = subplot_kws + if ndims == 1: + plotfunc = line + kwargs['hue'] = hue + elif ndims == 2: + if hue: + raise ValueError('hue is not compatible with 2d data') + plotfunc = pcolormesh else: - if row or col: + if row or col or hue: raise ValueError(error_msg) plotfunc = hist @@ -157,6 +185,61 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, rtol=0.01, return plotfunc(darray, **kwargs) +def _infer_line_data(darray, x, y, hue): + error_msg = ('must be either None or one of ({0:s})' + .format(', '.join([repr(dd) for dd in darray.dims]))) + ndims = len(darray.dims) + + if x is not None and x not in darray.dims: + raise ValueError('x ' + error_msg) + + if y is not None and y not in darray.dims: + raise ValueError('y ' + error_msg) + + if x is not None and y is not None: + raise ValueError('You cannot specify both x and y kwargs' + 'for line plots.') + + if ndims == 1: + dim, = darray.dims # get the only dimension name + huename = None + hueplt = None + huelabel = '' + + if (x is None and y is None) or x == dim: + xplt = darray.coords[dim] + yplt = darray + + else: + yplt = darray.coords[dim] + xplt = darray + + else: + if x is None and y is None and hue is None: + raise ValueError('For 2D inputs, please' + 'specify either hue, x or y.') + + if y is None: + xname, huename = _infer_xy_labels(darray=darray, x=x, y=hue) + yname = darray.name + xplt = darray.coords[xname] + yplt = darray.transpose(xname, huename) + + else: + yname, huename = _infer_xy_labels(darray=darray, x=y, y=hue) + xname = darray.name + xplt = darray.transpose(yname, huename) + yplt = darray.coords[yname] + + hueplt = darray.coords[huename] + huelabel = label_from_attrs(darray[huename]) + + xlabel = label_from_attrs(xplt) + ylabel = label_from_attrs(yplt) + + return xplt, yplt, hueplt, xlabel, ylabel, huelabel + + # This function signature should not change so that it can use # matplotlib format strings def line(darray, *args, **kwargs): @@ -182,8 +265,7 @@ def line(darray, *args, **kwargs): Axis on which to plot this figure. By default, use the current axis. Mutually exclusive with ``size`` and ``figsize``. hue : string, optional - Coordinate for which you want multiple lines plotted - (2D DataArrays only). + Coordinate for which you want multiple lines plotted. x, y : string, optional Coordinates for x, y axis. Only one of these may be specified. The other coordinate plots values from the DataArray on which this @@ -201,6 +283,15 @@ def line(darray, *args, **kwargs): """ + # Handle facetgrids first + row = kwargs.pop('row', None) + col = kwargs.pop('col', None) + if row or col: + allargs = locals().copy() + allargs.update(allargs.pop('kwargs')) + allargs.update(allargs.pop('args')) + return _line_facetgrid(**allargs) + ndims = len(darray.dims) if ndims > 2: raise ValueError('Line plots are for 1- or 2-dimensional DataArrays. ' @@ -218,70 +309,28 @@ def line(darray, *args, **kwargs): xincrease = kwargs.pop('xincrease', True) yincrease = kwargs.pop('yincrease', True) add_legend = kwargs.pop('add_legend', True) + _labels = kwargs.pop('_labels', True) ax = get_axis(figsize, size, aspect, ax) - - error_msg = ('must be either None or one of ({0:s})' - .format(', '.join([repr(dd) for dd in darray.dims]))) - - if x is not None and x not in darray.dims: - raise ValueError('x ' + error_msg) - - if y is not None and y not in darray.dims: - raise ValueError('y ' + error_msg) - - if x is not None and y is not None: - raise ValueError('You cannot specify both x and y kwargs' - 'for line plots.') - - if ndims == 1: - dim, = darray.dims # get the only dimension name - - if (x is None and y is None) or x == dim: - xplt = darray.coords[dim] - yplt = darray - - else: - yplt = darray.coords[dim] - xplt = darray - - else: - if x is None and y is None and hue is None: - raise ValueError('For 2D inputs, please specify either hue or x.') - - if y is None: - xlabel, huelabel = _infer_xy_labels(darray=darray, x=x, y=hue) - ylabel = darray.name - xplt = darray.coords[xlabel] - yplt = darray.transpose(xlabel, huelabel) - - else: - ylabel, huelabel = _infer_xy_labels(darray=darray, x=y, y=hue) - xlabel = darray.name - xplt = darray.transpose(ylabel, huelabel) - yplt = darray.coords[ylabel] - - huecoords = darray[huelabel] - huelabel = label_from_attrs(huecoords) - - xlabel = label_from_attrs(xplt) - ylabel = label_from_attrs(yplt) + xplt, yplt, hueplt, xlabel, ylabel, huelabel = \ + _infer_line_data(darray, x, y, hue) _ensure_plottable(xplt) primitive = ax.plot(xplt, yplt, *args, **kwargs) - if xlabel is not None: - ax.set_xlabel(xlabel) + if _labels: + if xlabel is not None: + ax.set_xlabel(xlabel) - if ylabel is not None: - ax.set_ylabel(ylabel) + if ylabel is not None: + ax.set_ylabel(ylabel) - ax.set_title(darray._title_for_slice()) + ax.set_title(darray._title_for_slice()) if darray.ndim == 2 and add_legend: ax.legend(handles=primitive, - labels=list(huecoords.values), + labels=list(hueplt.values), title=huelabel) # Rotate dates on xlabels diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index db1fb2fd081..cdb515ba92e 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1521,6 +1521,72 @@ def test_default_labels(self): assert substring_in_axes(label, ax) +class TestFacetedLinePlots(PlotTestCase): + def setUp(self): + self.darray = DataArray(np.random.randn(10, 6, 3, 4), + dims=['hue', 'x', 'col', 'row'], + coords=[range(10), range(6), + range(3), ['A', 'B', 'C', 'C++']], + name='Cornelius Ortega the 1st') + + def test_facetgrid_shape(self): + g = self.darray.plot(row='row', col='col', hue='hue') + assert g.axes.shape == (len(self.darray.row), len(self.darray.col)) + + g = self.darray.plot(row='col', col='row', hue='hue') + assert g.axes.shape == (len(self.darray.col), len(self.darray.row)) + + def test_default_labels(self): + g = self.darray.plot(row='row', col='col', hue='hue') + # Rightmost column should be labeled + for label, ax in zip(self.darray.coords['row'].values, g.axes[:, -1]): + assert substring_in_axes(label, ax) + + # Top row should be labeled + for label, ax in zip(self.darray.coords['col'].values, g.axes[0, :]): + assert substring_in_axes(str(label), ax) + + # Leftmost column should have array name + for ax in g.axes[:, 0]: + assert substring_in_axes(self.darray.name, ax) + + def test_test_empty_cell(self): + g = self.darray.isel(row=1).drop('row').plot(col='col', + hue='hue', + col_wrap=2) + bottomright = g.axes[-1, -1] + assert not bottomright.has_data() + assert not bottomright.get_visible() + + def test_set_axis_labels(self): + g = self.darray.plot(row='row', col='col', hue='hue') + g.set_axis_labels('longitude', 'latitude') + alltxt = text_in_fig() + + assert 'longitude' in alltxt + assert 'latitude' in alltxt + + def test_both_x_and_y(self): + with pytest.raises(ValueError): + self.darray.plot.line(row='row', col='col', + x='x', y='hue') + + def test_axes_in_faceted_plot(self): + with pytest.raises(ValueError): + self.darray.plot.line(row='row', col='col', + x='x', ax=plt.axes()) + + def test_figsize_and_size(self): + with pytest.raises(ValueError): + self.darray.plot.line(row='row', col='col', + x='x', size=3, figsize=4) + + def test_wrong_num_of_dimensions(self): + with pytest.raises(ValueError): + self.darray.plot(row='row', hue='hue') + self.darray.plot.line(row='row', hue='hue') + + class TestDatetimePlot(PlotTestCase): def setUp(self): ''' From 21a9f3d7e3a5dd729aeafd08dda966c365520965 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 7 Jun 2018 14:02:55 -0400 Subject: [PATCH 145/282] Feature/pickle rasterio (#2131) * add regression test * add PickleByReconstructionWrapper * docs * load in context manager * add distributed integration test * add test_pickle_reconstructor * drop lazy opening/caching and use partial function for open * stop using clever getattr hack * allow_cleanup_failure=ON_WINDOWS in tests for windows * whats new fix * fix bug in multiple pickles * fix for windows --- doc/whats-new.rst | 4 +++ xarray/backends/common.py | 28 ++++++++++++++++ xarray/backends/rasterio_.py | 56 +++++++++++++++++--------------- xarray/tests/test_backends.py | 40 +++++++++++++++++++++-- xarray/tests/test_distributed.py | 18 ++++++++-- 5 files changed, 114 insertions(+), 32 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 5e5da295186..980f996cb6d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -41,6 +41,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed a bug in ``rasterio`` backend which prevented use with ``distributed``. + The ``rasterio`` backend now returns pickleable objects (:issue:`2021`). + .. _whats-new.0.10.6: v0.10.6 (31 May 2018) @@ -220,6 +223,7 @@ Bug fixes By `Deepak Cherian `_. - Colorbar limits are now determined by excluding ±Infs too. By `Deepak Cherian `_. + By `Joe Hamman `_. - Fixed ``to_iris`` to maintain lazy dask array after conversion (:issue:`2046`). By `Alex Hilson `_ and `Stephan Hoyer `_. diff --git a/xarray/backends/common.py b/xarray/backends/common.py index 2961838e85f..d5eccd9be52 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -8,6 +8,7 @@ import traceback import warnings from collections import Mapping, OrderedDict +from functools import partial import numpy as np @@ -507,3 +508,30 @@ def assert_open(self): if not self._isopen: raise AssertionError('internal failure: file must be open ' 'if `autoclose=True` is used.') + + +class PickleByReconstructionWrapper(object): + + def __init__(self, opener, file, mode='r', **kwargs): + self.opener = partial(opener, file, mode=mode, **kwargs) + self.mode = mode + self._ds = None + + @property + def value(self): + self._ds = self.opener() + return self._ds + + def __getstate__(self): + state = self.__dict__.copy() + del state['_ds'] + if self.mode == 'w': + # file has already been created, don't override when restoring + state['mode'] = 'a' + return state + + def __setstate__(self, state): + self.__dict__.update(state) + + def close(self): + self._ds.close() diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 8c0764c3ec9..0f19a1b51be 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -8,7 +8,7 @@ from .. import DataArray from ..core import indexing from ..core.utils import is_scalar -from .common import BackendArray +from .common import BackendArray, PickleByReconstructionWrapper try: from dask.utils import SerializableLock as Lock @@ -25,15 +25,15 @@ class RasterioArrayWrapper(BackendArray): """A wrapper around rasterio dataset objects""" - def __init__(self, rasterio_ds): - self.rasterio_ds = rasterio_ds - self._shape = (rasterio_ds.count, rasterio_ds.height, - rasterio_ds.width) + def __init__(self, riods): + self.riods = riods + self._shape = (riods.value.count, riods.value.height, + riods.value.width) self._ndims = len(self.shape) @property def dtype(self): - dtypes = self.rasterio_ds.dtypes + dtypes = self.riods.value.dtypes if not np.all(np.asarray(dtypes) == dtypes[0]): raise ValueError('All bands should have the same dtype') return np.dtype(dtypes[0]) @@ -105,7 +105,7 @@ def _get_indexer(self, key): def __getitem__(self, key): band_key, window, squeeze_axis, np_inds = self._get_indexer(key) - out = self.rasterio_ds.read(band_key, window=tuple(window)) + out = self.riods.value.read(band_key, window=tuple(window)) if squeeze_axis: out = np.squeeze(out, axis=squeeze_axis) return indexing.NumpyIndexingAdapter(out)[np_inds] @@ -194,7 +194,8 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, """ import rasterio - riods = rasterio.open(filename, mode='r') + + riods = PickleByReconstructionWrapper(rasterio.open, filename, mode='r') if cache is None: cache = chunks is None @@ -202,20 +203,20 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, coords = OrderedDict() # Get bands - if riods.count < 1: + if riods.value.count < 1: raise ValueError('Unknown dims') - coords['band'] = np.asarray(riods.indexes) + coords['band'] = np.asarray(riods.value.indexes) # Get coordinates if LooseVersion(rasterio.__version__) < '1.0': - transform = riods.affine + transform = riods.value.affine else: - transform = riods.transform + transform = riods.value.transform if transform.is_rectilinear: # 1d coordinates parse = True if parse_coordinates is None else parse_coordinates if parse: - nx, ny = riods.width, riods.height + nx, ny = riods.value.width, riods.value.height # xarray coordinates are pixel centered x, _ = (np.arange(nx) + 0.5, np.zeros(nx) + 0.5) * transform _, y = (np.zeros(ny) + 0.5, np.arange(ny) + 0.5) * transform @@ -238,41 +239,42 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, # For serialization store as tuple of 6 floats, the last row being # always (0, 0, 1) per definition (see https://github.com/sgillies/affine) attrs['transform'] = tuple(transform)[:6] - if hasattr(riods, 'crs') and riods.crs: + if hasattr(riods.value, 'crs') and riods.value.crs: # CRS is a dict-like object specific to rasterio # If CRS is not None, we convert it back to a PROJ4 string using # rasterio itself - attrs['crs'] = riods.crs.to_string() - if hasattr(riods, 'res'): + attrs['crs'] = riods.value.crs.to_string() + if hasattr(riods.value, 'res'): # (width, height) tuple of pixels in units of CRS - attrs['res'] = riods.res - if hasattr(riods, 'is_tiled'): + attrs['res'] = riods.value.res + if hasattr(riods.value, 'is_tiled'): # Is the TIF tiled? (bool) # We cast it to an int for netCDF compatibility - attrs['is_tiled'] = np.uint8(riods.is_tiled) + attrs['is_tiled'] = np.uint8(riods.value.is_tiled) with warnings.catch_warnings(): - # casting riods.transform to a tuple makes this future proof + # casting riods.value.transform to a tuple makes this future proof warnings.simplefilter('ignore', FutureWarning) - if hasattr(riods, 'transform'): + if hasattr(riods.value, 'transform'): # Affine transformation matrix (tuple of floats) # Describes coefficients mapping pixel coordinates to CRS - attrs['transform'] = tuple(riods.transform) - if hasattr(riods, 'nodatavals'): + attrs['transform'] = tuple(riods.value.transform) + if hasattr(riods.value, 'nodatavals'): # The nodata values for the raster bands attrs['nodatavals'] = tuple([np.nan if nodataval is None else nodataval - for nodataval in riods.nodatavals]) + for nodataval in riods.value.nodatavals]) # Parse extra metadata from tags, if supported parsers = {'ENVI': _parse_envi} - driver = riods.driver + driver = riods.value.driver if driver in parsers: - meta = parsers[driver](riods.tags(ns=driver)) + meta = parsers[driver](riods.value.tags(ns=driver)) for k, v in meta.items(): # Add values as coordinates if they match the band count, # as attributes otherwise - if isinstance(v, (list, np.ndarray)) and len(v) == riods.count: + if (isinstance(v, (list, np.ndarray)) and + len(v) == riods.value.count): coords[k] = ('band', np.asarray(v)) else: attrs[k] = v diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 0e6151b2db5..df7ed66f4fd 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -19,7 +19,8 @@ from xarray import ( DataArray, Dataset, backends, open_dataarray, open_dataset, open_mfdataset, save_mfdataset) -from xarray.backends.common import robust_getitem +from xarray.backends.common import (robust_getitem, + PickleByReconstructionWrapper) from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.backends.pydap_ import PydapDataStore from xarray.core import indexing @@ -2724,7 +2725,8 @@ def create_tmp_geotiff(nx=4, ny=3, nz=3, # yields a temporary geotiff file and a corresponding expected DataArray import rasterio from rasterio.transform import from_origin - with create_tmp_file(suffix='.tif') as tmp_file: + with create_tmp_file(suffix='.tif', + allow_cleanup_failure=ON_WINDOWS) as tmp_file: # allow 2d or 3d shapes if nz == 1: data_shape = ny, nx @@ -2996,6 +2998,14 @@ def test_chunks(self): ex = expected.sel(band=1).mean(dim='x') assert_allclose(ac, ex) + def test_pickle_rasterio(self): + # regression test for https://github.com/pydata/xarray/issues/2121 + with create_tmp_geotiff() as (tmp_file, expected): + with xr.open_rasterio(tmp_file) as rioda: + temp = pickle.dumps(rioda) + with pickle.loads(temp) as actual: + assert_equal(actual, rioda) + def test_ENVI_tags(self): rasterio = pytest.importorskip('rasterio', minversion='1.0a') from rasterio.transform import from_origin @@ -3260,3 +3270,29 @@ def test_dataarray_to_netcdf_no_name_pathlib(self): with open_dataarray(tmp) as loaded_da: assert_identical(original_da, loaded_da) + + +def test_pickle_reconstructor(): + + lines = ['foo bar spam eggs'] + + with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as tmp: + with open(tmp, 'w') as f: + f.writelines(lines) + + obj = PickleByReconstructionWrapper(open, tmp) + + assert obj.value.readlines() == lines + + p_obj = pickle.dumps(obj) + obj.value.close() # for windows + obj2 = pickle.loads(p_obj) + + assert obj2.value.readlines() == lines + + # roundtrip again to make sure we can fully restore the state + p_obj2 = pickle.dumps(obj2) + obj2.value.close() # for windows + obj3 = pickle.loads(p_obj2) + + assert obj3.value.readlines() == lines diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 0ac03327494..8679e892be4 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -17,13 +17,14 @@ from distributed.client import futures_of import xarray as xr -from xarray.tests.test_backends import ON_WINDOWS, create_tmp_file +from xarray.tests.test_backends import (ON_WINDOWS, create_tmp_file, + create_tmp_geotiff) from xarray.tests.test_dataset import create_test_data from xarray.backends.common import HDF5_LOCK, CombinedLock from . import ( - assert_allclose, has_h5netcdf, has_netCDF4, has_scipy, requires_zarr, - raises_regex) + assert_allclose, has_h5netcdf, has_netCDF4, requires_rasterio, has_scipy, + requires_zarr, raises_regex) # this is to stop isort throwing errors. May have been easier to just use # `isort:skip` in retrospect @@ -136,6 +137,17 @@ def test_dask_distributed_zarr_integration_test(loop): assert_allclose(original, computed) +@requires_rasterio +def test_dask_distributed_rasterio_integration_test(loop): + with create_tmp_geotiff() as (tmp_file, expected): + with cluster() as (s, [a, b]): + with Client(s['address'], loop=loop) as c: + da_tiff = xr.open_rasterio(tmp_file, chunks={'band': 1}) + assert isinstance(da_tiff.data, da.Array) + actual = da_tiff.compute() + assert_allclose(actual, expected) + + @pytest.mark.skipif(distributed.__version__ <= '1.19.3', reason='Need recent distributed version to clean up get') @gen_cluster(client=True, timeout=None) From e39729928544204894e65c187d66c1a2b1900fea Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 8 Jun 2018 09:33:51 +0900 Subject: [PATCH 146/282] implement interp() (#2104) * Start working * interp1d for numpy backed array. * interp1d for dask backed array. * Support scalar interpolation. * more docs * flake8. Remove an unnecessary file. * Remove non-unicode characters * refactoring... * flake8. whats new * Make tests skip if scipy is not installed * skipif -> skip * move skip into every function * remove reuires_scipy * refactoring exceptions. * assert_equal -> assert_allclose * Remove unintended word. * More tests. More docs. * More docs. * Added a benchmark * doc. Remove *.png file. * add .load to benchmark with dask. * add assume_sorted kwarg. * Support dimension without coordinate * flake8 * More docs. test for attrs. * Updates based on comments * rename test * update docs * Add transpose for python 2 * More strict ordering * Cleanup * Update doc * Add skipif in tests * minor grammar/language edits in docs * Support dict arguments for interp. * update based on comments * Remove unused if-block * ValueError -> NotImpletedError. Doc improvement * Using OrderedSet * Drop object array after interpolation. * flake8 * Add keep_attrs keyword * flake8 (reverted from commit 6e0099963a50dc622204a690a0058b4db527b8ef) * flake8 * Remove keep_attrs keywords * Returns copy for not-interpolated variable. * Fix docs --- asv_bench/benchmarks/interp.py | 54 ++ .../advanced_selection_interpolation.svg | 731 ++++++++++++++++++ doc/api.rst | 2 + doc/index.rst | 2 + doc/indexing.rst | 2 +- doc/installing.rst | 1 + doc/interpolation.rst | 261 +++++++ doc/whats-new.rst | 10 + xarray/core/computation.py | 2 +- xarray/core/dataarray.py | 46 +- xarray/core/dataset.py | 89 ++- xarray/core/missing.py | 222 +++++- xarray/tests/test_interp.py | 432 +++++++++++ 13 files changed, 1840 insertions(+), 14 deletions(-) create mode 100644 asv_bench/benchmarks/interp.py create mode 100644 doc/_static/advanced_selection_interpolation.svg create mode 100644 doc/interpolation.rst create mode 100644 xarray/tests/test_interp.py diff --git a/asv_bench/benchmarks/interp.py b/asv_bench/benchmarks/interp.py new file mode 100644 index 00000000000..edec6df34dd --- /dev/null +++ b/asv_bench/benchmarks/interp.py @@ -0,0 +1,54 @@ +from __future__ import absolute_import, division, print_function + +import numpy as np +import pandas as pd + +import xarray as xr + +from . import parameterized, randn, requires_dask + +nx = 3000 +long_nx = 30000000 +ny = 2000 +nt = 1000 +window = 20 + +randn_xy = randn((nx, ny), frac_nan=0.1) +randn_xt = randn((nx, nt)) +randn_t = randn((nt, )) +randn_long = randn((long_nx, ), frac_nan=0.1) + + +new_x_short = np.linspace(0.3 * nx, 0.7 * nx, 100) +new_x_long = np.linspace(0.3 * nx, 0.7 * nx, 1000) +new_y_long = np.linspace(0.1, 0.9, 1000) + + +class Interpolation(object): + def setup(self, *args, **kwargs): + self.ds = xr.Dataset( + {'var1': (('x', 'y'), randn_xy), + 'var2': (('x', 't'), randn_xt), + 'var3': (('t', ), randn_t)}, + coords={'x': np.arange(nx), + 'y': np.linspace(0, 1, ny), + 't': pd.date_range('1970-01-01', periods=nt, freq='D'), + 'x_coords': ('x', np.linspace(1.1, 2.1, nx))}) + + @parameterized(['method', 'is_short'], + (['linear', 'cubic'], [True, False])) + def time_interpolation(self, method, is_short): + new_x = new_x_short if is_short else new_x_long + self.ds.interp(x=new_x, method=method).load() + + @parameterized(['method'], + (['linear', 'nearest'])) + def time_interpolation_2d(self, method): + self.ds.interp(x=new_x_long, y=new_y_long, method=method).load() + + +class InterpolationDask(Interpolation): + def setup(self, *args, **kwargs): + requires_dask() + super(InterpolationDask, self).setup(**kwargs) + self.ds = self.ds.chunk({'t': 50}) diff --git a/doc/_static/advanced_selection_interpolation.svg b/doc/_static/advanced_selection_interpolation.svg new file mode 100644 index 00000000000..096563a604f --- /dev/null +++ b/doc/_static/advanced_selection_interpolation.svg @@ -0,0 +1,731 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + y + x + + + + + z + + + + + + + + + + + + + + + + + + + + + + + + + + + + y + x + + + + + z + + + + + + + + + Advanced indexing + Advanced interpolation + + + + diff --git a/doc/api.rst b/doc/api.rst index a528496bb6a..cb44ef82c8f 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -110,6 +110,7 @@ Indexing Dataset.isel Dataset.sel Dataset.squeeze + Dataset.interp Dataset.reindex Dataset.reindex_like Dataset.set_index @@ -263,6 +264,7 @@ Indexing DataArray.isel DataArray.sel DataArray.squeeze + DataArray.interp DataArray.reindex DataArray.reindex_like DataArray.set_index diff --git a/doc/index.rst b/doc/index.rst index dc00c548b35..7528f3cb1fa 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -40,6 +40,7 @@ Documentation * :doc:`data-structures` * :doc:`indexing` +* :doc:`interpolation` * :doc:`computation` * :doc:`groupby` * :doc:`reshaping` @@ -57,6 +58,7 @@ Documentation data-structures indexing + interpolation computation groupby reshaping diff --git a/doc/indexing.rst b/doc/indexing.rst index cec438dd2e4..a44e64e4079 100644 --- a/doc/indexing.rst +++ b/doc/indexing.rst @@ -510,7 +510,7 @@ where three elements at ``(ix, iy) = ((0, 0), (1, 1), (6, 0))`` are selected and mapped along a new dimension ``z``. If you want to add a coordinate to the new dimension ``z``, -you can supply a :py:meth:`~xarray.DataArray` with a coordinate, +you can supply a :py:class:`~xarray.DataArray` with a coordinate, .. ipython:: python diff --git a/doc/installing.rst b/doc/installing.rst index 33f01b8c770..31fc109ee2e 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -35,6 +35,7 @@ For netCDF and IO For accelerating xarray ~~~~~~~~~~~~~~~~~~~~~~~ +- `scipy `__: necessary to enable the interpolation features for xarray objects - `bottleneck `__: speeds up NaN-skipping and rolling window aggregations by a large factor (1.1 or later) diff --git a/doc/interpolation.rst b/doc/interpolation.rst new file mode 100644 index 00000000000..c5fd5166aeb --- /dev/null +++ b/doc/interpolation.rst @@ -0,0 +1,261 @@ +.. _interp: + +Interpolating data +================== + +.. ipython:: python + :suppress: + + import numpy as np + import pandas as pd + import xarray as xr + np.random.seed(123456) + +xarray offers flexible interpolation routines, which have a similar interface +to our :ref:`indexing `. + +.. note:: + + ``interp`` requires `scipy` installed. + + +Scalar and 1-dimensional interpolation +-------------------------------------- + +Interpolating a :py:class:`~xarray.DataArray` works mostly like labeled +indexing of a :py:class:`~xarray.DataArray`, + +.. ipython:: python + + da = xr.DataArray(np.sin(0.3 * np.arange(12).reshape(4, 3)), + [('time', np.arange(4)), + ('space', [0.1, 0.2, 0.3])]) + # label lookup + da.sel(time=3) + + # interpolation + da.interp(time=3.5) + + +Similar to the indexing, :py:meth:`~xarray.DataArray.interp` also accepts an +array-like, which gives the interpolated result as an array. + +.. ipython:: python + + # label lookup + da.sel(time=[2, 3]) + + # interpolation + da.interp(time=[2.5, 3.5]) + +.. note:: + + Currently, our interpolation only works for regular grids. + Therefore, similarly to :py:meth:`~xarray.DataArray.sel`, + only 1D coordinates along a dimension can be used as the + original coordinate to be interpolated. + + +Multi-dimensional Interpolation +------------------------------- + +Like :py:meth:`~xarray.DataArray.sel`, :py:meth:`~xarray.DataArray.interp` +accepts multiple coordinates. In this case, multidimensional interpolation +is carried out. + +.. ipython:: python + + # label lookup + da.sel(time=2, space=0.1) + + # interpolation + da.interp(time=2.5, space=0.15) + +Array-like coordinates are also accepted: + +.. ipython:: python + + # label lookup + da.sel(time=[2, 3], space=[0.1, 0.2]) + + # interpolation + da.interp(time=[1.5, 2.5], space=[0.15, 0.25]) + + +Interpolation methods +--------------------- + +We use :py:func:`scipy.interpolate.interp1d` for 1-dimensional interpolation and +:py:func:`scipy.interpolate.interpn` for multi-dimensional interpolation. + +The interpolation method can be specified by the optional ``method`` argument. + +.. ipython:: python + + da = xr.DataArray(np.sin(np.linspace(0, 2 * np.pi, 10)), dims='x', + coords={'x': np.linspace(0, 1, 10)}) + + da.plot.line('o', label='original') + da.interp(x=np.linspace(0, 1, 100)).plot.line(label='linear (default)') + da.interp(x=np.linspace(0, 1, 100), method='cubic').plot.line(label='cubic') + @savefig interpolation_sample1.png width=4in + plt.legend() + +Additional keyword arguments can be passed to scipy's functions. + +.. ipython:: python + + # fill 0 for the outside of the original coordinates. + da.interp(x=np.linspace(-0.5, 1.5, 10), kwargs={'fill_value': 0.0}) + # extrapolation + da.interp(x=np.linspace(-0.5, 1.5, 10), kwargs={'fill_value': 'extrapolate'}) + + +Advanced Interpolation +---------------------- + +:py:meth:`~xarray.DataArray.interp` accepts :py:class:`~xarray.DataArray` +as similar to :py:meth:`~xarray.DataArray.sel`, which enables us more advanced interpolation. +Based on the dimension of the new coordinate passed to :py:meth:`~xarray.DataArray.interp`, the dimension of the result are determined. + +For example, if you want to interpolate a two dimensional array along a particular dimension, as illustrated below, +you can pass two 1-dimensional :py:class:`~xarray.DataArray` s with +a common dimension as new coordinate. + +.. image:: _static/advanced_selection_interpolation.svg + :height: 200px + :width: 400 px + :alt: advanced indexing and interpolation + :align: center + +For example: + +.. ipython:: python + + da = xr.DataArray(np.sin(0.3 * np.arange(20).reshape(5, 4)), + [('x', np.arange(5)), + ('y', [0.1, 0.2, 0.3, 0.4])]) + # advanced indexing + x = xr.DataArray([0, 2, 4], dims='z') + y = xr.DataArray([0.1, 0.2, 0.3], dims='z') + da.sel(x=x, y=y) + + # advanced interpolation + x = xr.DataArray([0.5, 1.5, 2.5], dims='z') + y = xr.DataArray([0.15, 0.25, 0.35], dims='z') + da.interp(x=x, y=y) + +where values on the original coordinates +``(x, y) = ((0.5, 0.15), (1.5, 0.25), (2.5, 0.35))`` are obtained by the +2-dimensional interpolation and mapped along a new dimension ``z``. + +If you want to add a coordinate to the new dimension ``z``, you can supply +:py:class:`~xarray.DataArray` s with a coordinate, + +.. ipython:: python + + x = xr.DataArray([0.5, 1.5, 2.5], dims='z', coords={'z': ['a', 'b','c']}) + y = xr.DataArray([0.15, 0.25, 0.35], dims='z', + coords={'z': ['a', 'b','c']}) + da.interp(x=x, y=y) + +For the details of the advanced indexing, +see :ref:`more advanced indexing `. + + +Interpolating arrays with NaN +----------------------------- + +Our :py:meth:`~xarray.DataArray.interp` works with arrays with NaN +the same way that +`scipy.interpolate.interp1d `_ and +`scipy.interpolate.interpn `_ do. +``linear`` and ``nearest`` methods return arrays including NaN, +while other methods such as ``cubic`` or ``quadratic`` return all NaN arrays. + +.. ipython:: python + + da = xr.DataArray([0, 2, np.nan, 3, 3.25], dims='x', + coords={'x': range(5)}) + da.interp(x=[0.5, 1.5, 2.5]) + da.interp(x=[0.5, 1.5, 2.5], method='cubic') + +To avoid this, you can drop NaN by :py:meth:`~xarray.DataArray.dropna`, and +then make the interpolation + +.. ipython:: python + + dropped = da.dropna('x') + dropped + dropped.interp(x=[0.5, 1.5, 2.5], method='cubic') + +If NaNs are distributed rondomly in your multidimensional array, +dropping all the columns containing more than one NaNs by +:py:meth:`~xarray.DataArray.dropna` may lose a significant amount of information. +In such a case, you can fill NaN by :py:meth:`~xarray.DataArray.interpolate_na`, +which is similar to :py:meth:`pandas.Series.interpolate`. + +.. ipython:: python + + filled = da.interpolate_na(dim='x') + filled + +This fills NaN by interpolating along the specified dimension. +After filling NaNs, you can interpolate: + +.. ipython:: python + + filled.interp(x=[0.5, 1.5, 2.5], method='cubic') + +For the details of :py:meth:`~xarray.DataArray.interpolate_na`, +see :ref:`Missing values `. + + +Example +------- + +Let's see how :py:meth:`~xarray.DataArray.interp` works on real data. + +.. ipython:: python + + # Raw data + ds = xr.tutorial.load_dataset('air_temperature') + fig, axes = plt.subplots(ncols=2, figsize=(10, 4)) + ds.air.isel(time=0).plot(ax=axes[0]) + axes[0].set_title('Raw data') + + # Interpolated data + new_lon = np.linspace(ds.lon[0], ds.lon[-1], ds.dims['lon'] * 4) + new_lat = np.linspace(ds.lat[0], ds.lat[-1], ds.dims['lat'] * 4) + dsi = ds.interp(lat=new_lat, lon=new_lon) + dsi.air.isel(time=0).plot(ax=axes[1]) + @savefig interpolation_sample3.png width=8in + axes[1].set_title('Interpolated data') + +Our advanced interpolation can be used to remap the data to the new coordinate. +Consider the new coordinates x and z on the two dimensional plane. +The remapping can be done as follows + +.. ipython:: python + + # new coordinate + x = np.linspace(240, 300, 100) + z = np.linspace(20, 70, 100) + # relation between new and original coordinates + lat = xr.DataArray(z, dims=['z'], coords={'z': z}) + lon = xr.DataArray((x[:, np.newaxis]-270)/np.cos(z*np.pi/180)+270, + dims=['x', 'z'], coords={'x': x, 'z': z}) + + fig, axes = plt.subplots(ncols=2, figsize=(10, 4)) + ds.air.isel(time=0).plot(ax=axes[0]) + # draw the new coordinate on the original coordinates. + for idx in [0, 33, 66, 99]: + axes[0].plot(lon.isel(x=idx), lat, '--k') + for idx in [0, 33, 66, 99]: + axes[0].plot(*xr.broadcast(lon.isel(z=idx), lat.isel(z=idx)), '--k') + axes[0].set_title('Raw data') + + dsi = ds.interp(lon=lon, lat=lat) + dsi.air.isel(time=0).plot(ax=axes[1]) + @savefig interpolation_sample4.png width=8in + axes[1].set_title('Remapped data') diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 980f996cb6d..44f829874ac 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -38,6 +38,15 @@ Enhancements - Plot labels now make use of metadata that follow CF conventions. By `Deepak Cherian `_ and `Ryan Abernathey `_. +- :py:meth:`~xarray.DataArray.interp` and :py:meth:`~xarray.Dataset.interp` + methods are newly added. + See :ref:`interpolating values with interp` for the detail. + (:issue:`2079`) + By `Keisuke Fujii `_. + +- `:py:meth:`~DataArray.dot` and :py:func:`~dot` are partly supported with older + dask<0.17.4. (related to :issue:`2203`) + By `Keisuke Fujii `_. + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 6a49610cb7b..9b251bb2c4b 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -10,7 +10,7 @@ import numpy as np -from . import duck_array_ops, utils, dtypes +from . import duck_array_ops, utils from .alignment import deep_align from .merge import expand_and_merge_variables from .pycompat import OrderedDict, dask_array_type, basestring diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index fd2b49cc08a..4129a3c5f26 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -906,10 +906,54 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, indexers=indexers, method=method, tolerance=tolerance, copy=copy) return self._from_temp_dataset(ds) + def interp(self, coords=None, method='linear', assume_sorted=False, + kwargs={}, **coords_kwargs): + """ Multidimensional interpolation of variables. + + coords : dict, optional + Mapping from dimension names to the new coordinates. + new coordinate can be an scalar, array-like or DataArray. + If DataArrays are passed as new coordates, their dimensions are + used for the broadcasting. + method: {'linear', 'nearest'} for multidimensional array, + {'linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic'} + for 1-dimensional array. + assume_sorted: boolean, optional + If False, values of x can be in any order and they are sorted + first. If True, x has to be an array of monotonically increasing + values. + kwargs: dictionary + Additional keyword passed to scipy's interpolator. + **coords_kwarg : {dim: coordinate, ...}, optional + The keyword arguments form of ``coords``. + One of coords or coords_kwargs must be provided. + + Returns + ------- + interpolated: xr.DataArray + New dataarray on the new coordinates. + + Note + ---- + scipy is required. + + See Also + -------- + scipy.interpolate.interp1d + scipy.interpolate.interpn + """ + if self.dtype.kind not in 'uifc': + raise TypeError('interp only works for a numeric type array. ' + 'Given {}.'.format(self.dtype)) + + ds = self._to_temp_dataset().interp( + coords, method=method, kwargs=kwargs, assume_sorted=assume_sorted, + **coords_kwargs) + return self._from_temp_dataset(ds) + def rename(self, new_name_or_name_dict=None, **names): """Returns a new DataArray with renamed coordinates or a new name. - Parameters ---------- new_name_or_name_dict : str or dict-like, optional diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 08f5f70d72b..90712c953da 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1318,7 +1318,7 @@ def _validate_indexers(self, indexers): # all indexers should be int, slice, np.ndarrays, or Variable indexers_list = [] for k, v in iteritems(indexers): - if isinstance(v, integer_types + (slice, Variable)): + if isinstance(v, (slice, Variable)): pass elif isinstance(v, DataArray): v = v.variable @@ -1328,6 +1328,14 @@ def _validate_indexers(self, indexers): raise TypeError('cannot use a Dataset as an indexer') else: v = np.asarray(v) + if v.ndim == 0: + v = as_variable(v) + elif v.ndim == 1: + v = as_variable((k, v)) + else: + raise IndexError( + "Unlabeled multi-dimensional array cannot be " + "used for indexing: {}".format(k)) indexers_list.append((k, v)) return indexers_list @@ -1806,6 +1814,85 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, coord_names.update(indexers) return self._replace_vars_and_dims(variables, coord_names) + def interp(self, coords=None, method='linear', assume_sorted=False, + kwargs={}, **coords_kwargs): + """ Multidimensional interpolation of Dataset. + + Parameters + ---------- + coords : dict, optional + Mapping from dimension names to the new coordinates. + New coordinate can be a scalar, array-like or DataArray. + If DataArrays are passed as new coordates, their dimensions are + used for the broadcasting. + method: string, optional. + {'linear', 'nearest'} for multidimensional array, + {'linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic'} + for 1-dimensional array. 'linear' is used by default. + assume_sorted: boolean, optional + If False, values of coordinates that are interpolated over can be + in any order and they are sorted first. If True, interpolated + coordinates are assumed to be an array of monotonically increasing + values. + kwargs: dictionary, optional + Additional keyword passed to scipy's interpolator. + **coords_kwarg : {dim: coordinate, ...}, optional + The keyword arguments form of ``coords``. + One of coords or coords_kwargs must be provided. + + Returns + ------- + interpolated: xr.Dataset + New dataset on the new coordinates. + + Note + ---- + scipy is required. + + See Also + -------- + scipy.interpolate.interp1d + scipy.interpolate.interpn + """ + from . import missing + + coords = either_dict_or_kwargs(coords, coords_kwargs, 'rename') + indexers = OrderedDict(self._validate_indexers(coords)) + + obj = self if assume_sorted else self.sortby([k for k in coords]) + + def maybe_variable(obj, k): + # workaround to get variable for dimension without coordinate. + try: + return obj._variables[k] + except KeyError: + return as_variable((k, range(obj.dims[k]))) + + variables = OrderedDict() + for name, var in iteritems(obj._variables): + if name not in indexers: + if var.dtype.kind in 'uifc': + var_indexers = {k: (maybe_variable(obj, k), v) for k, v + in indexers.items() if k in var.dims} + variables[name] = missing.interp( + var, var_indexers, method, **kwargs) + elif all(d not in indexers for d in var.dims): + # keep unrelated object array + variables[name] = var + + coord_names = set(variables).intersection(obj._coord_names) + selected = obj._replace_vars_and_dims(variables, + coord_names=coord_names) + # attach indexer as coordinate + variables.update(indexers) + # Extract coordinates from indexers + coord_vars = selected._get_indexers_coordinates(coords) + variables.update(coord_vars) + coord_names = (set(variables) + .intersection(obj._coord_names) + .union(coord_vars)) + return obj._replace_vars_and_dims(variables, coord_names=coord_names) + def rename(self, name_dict=None, inplace=False, **names): """Returns a new object with renamed variables and dimensions. diff --git a/xarray/core/missing.py b/xarray/core/missing.py index 0da6750f5bc..e10f37d58d8 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -10,7 +10,9 @@ from .computation import apply_ufunc from .npcompat import flip from .pycompat import iteritems -from .utils import is_scalar +from .utils import is_scalar, OrderedSet +from .variable import Variable, broadcast_variables +from .duck_array_ops import dask_array_type class BaseInterpolator(object): @@ -203,7 +205,8 @@ def interp_na(self, dim=None, use_coordinate=True, method='linear', limit=None, # method index = get_clean_interp_index(self, dim, use_coordinate=use_coordinate, **kwargs) - interpolator = _get_interpolator(method, **kwargs) + interp_class, kwargs = _get_interpolator(method, **kwargs) + interpolator = partial(func_interpolate_na, interp_class, **kwargs) arr = apply_ufunc(interpolator, index, self, input_core_dims=[[dim], [dim]], @@ -219,7 +222,7 @@ def interp_na(self, dim=None, use_coordinate=True, method='linear', limit=None, return arr -def wrap_interpolator(interpolator, x, y, **kwargs): +def func_interpolate_na(interpolator, x, y, **kwargs): '''helper function to apply interpolation along 1 dimension''' # it would be nice if this wasn't necessary, works around: # "ValueError: assignment destination is read-only" in assignment below @@ -281,29 +284,41 @@ def bfill(arr, dim=None, limit=None): kwargs=dict(n=_limit, axis=axis)).transpose(*arr.dims) -def _get_interpolator(method, **kwargs): +def _get_interpolator(method, vectorizeable_only=False, **kwargs): '''helper function to select the appropriate interpolator class - returns a partial of wrap_interpolator + returns interpolator class and keyword arguments for the class ''' interp1d_methods = ['linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic', 'polynomial'] valid_methods = interp1d_methods + ['barycentric', 'krog', 'pchip', 'spline', 'akima'] + has_scipy = True + try: + from scipy import interpolate + except ImportError: + has_scipy = False + + # prioritize scipy.interpolate if (method == 'linear' and not - kwargs.get('fill_value', None) == 'extrapolate'): + kwargs.get('fill_value', None) == 'extrapolate' and + not vectorizeable_only): kwargs.update(method=method) interp_class = NumpyInterpolator + elif method in valid_methods: - try: - from scipy import interpolate - except ImportError: + if not has_scipy: raise ImportError( 'Interpolation with method `%s` requires scipy' % method) + if method in interp1d_methods: kwargs.update(method=method) interp_class = ScipyInterpolator + elif vectorizeable_only: + raise ValueError('{} is not a vectorizeable interpolator. ' + 'Available methods are {}'.format( + method, interp1d_methods)) elif method == 'barycentric': interp_class = interpolate.BarycentricInterpolator elif method == 'krog': @@ -320,7 +335,30 @@ def _get_interpolator(method, **kwargs): else: raise ValueError('%s is not a valid interpolator' % method) - return partial(wrap_interpolator, interp_class, **kwargs) + return interp_class, kwargs + + +def _get_interpolator_nd(method, **kwargs): + '''helper function to select the appropriate interpolator class + + returns interpolator class and keyword arguments for the class + ''' + valid_methods = ['linear', 'nearest'] + + try: + from scipy import interpolate + except ImportError: + raise ImportError( + 'Interpolation with method `%s` requires scipy' % method) + + if method in valid_methods: + kwargs.update(method=method) + interp_class = interpolate.interpn + else: + raise ValueError('%s is not a valid interpolator for interpolating ' + 'over multiple dimensions.' % method) + + return interp_class, kwargs def _get_valid_fill_mask(arr, dim, limit): @@ -332,3 +370,167 @@ def _get_valid_fill_mask(arr, dim, limit): return (arr.isnull().rolling(min_periods=1, **kw) .construct(new_dim, fill_value=False) .sum(new_dim, skipna=False)) <= limit + + +def _assert_single_chunk(var, axes): + for axis in axes: + if len(var.chunks[axis]) > 1 or var.chunks[axis][0] < var.shape[axis]: + raise NotImplementedError( + 'Chunking along the dimension to be interpolated ' + '({}) is not yet supported.'.format(axis)) + + +def _localize(var, indexes_coords): + """ Speed up for linear and nearest neighbor method. + Only consider a subspace that is needed for the interpolation + """ + indexes = {} + for dim, [x, new_x] in indexes_coords.items(): + index = x.to_index() + imin = index.get_loc(np.min(new_x.values), method='nearest') + imax = index.get_loc(np.max(new_x.values), method='nearest') + + indexes[dim] = slice(max(imin - 2, 0), imax + 2) + indexes_coords[dim] = (x[indexes[dim]], new_x) + return var.isel(**indexes), indexes_coords + + +def interp(var, indexes_coords, method, **kwargs): + """ Make an interpolation of Variable + + Parameters + ---------- + var: Variable + index_coords: + Mapping from dimension name to a pair of original and new coordinates. + Original coordinates should be sorted in strictly ascending order. + Note that all the coordinates should be Variable objects. + method: string + One of {'linear', 'nearest', 'zero', 'slinear', 'quadratic', + 'cubic'}. For multidimensional interpolation, only + {'linear', 'nearest'} can be used. + **kwargs: + keyword arguments to be passed to scipy.interpolate + + Returns + ------- + Interpolated Variable + + See Also + -------- + DataArray.interp + Dataset.interp + """ + if not indexes_coords: + return var.copy() + + # simple speed up for the local interpolation + if method in ['linear', 'nearest']: + var, indexes_coords = _localize(var, indexes_coords) + + # default behavior + kwargs['bounds_error'] = kwargs.get('bounds_error', False) + + # target dimensions + dims = list(indexes_coords) + x, new_x = zip(*[indexes_coords[d] for d in dims]) + destination = broadcast_variables(*new_x) + + # transpose to make the interpolated axis to the last position + broadcast_dims = [d for d in var.dims if d not in dims] + original_dims = broadcast_dims + dims + new_dims = broadcast_dims + list(destination[0].dims) + interped = interp_func(var.transpose(*original_dims).data, + x, destination, method, kwargs) + + result = Variable(new_dims, interped, attrs=var.attrs) + + # dimension of the output array + out_dims = OrderedSet() + for d in var.dims: + if d in dims: + out_dims.update(indexes_coords[d][1].dims) + else: + out_dims.add(d) + return result.transpose(*tuple(out_dims)) + + +def interp_func(var, x, new_x, method, kwargs): + """ + multi-dimensional interpolation for array-like. Interpolated axes should be + located in the last position. + + Parameters + ---------- + var: np.ndarray or dask.array.Array + Array to be interpolated. The final dimension is interpolated. + x: a list of 1d array. + Original coordinates. Should not contain NaN. + new_x: a list of 1d array + New coordinates. Should not contain NaN. + method: string + {'linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic'} for + 1-dimensional itnterpolation. + {'linear', 'nearest'} for multidimensional interpolation + **kwargs: + Optional keyword arguments to be passed to scipy.interpolator + + Returns + ------- + interpolated: array + Interpolated array + + Note + ---- + This requiers scipy installed. + + See Also + -------- + scipy.interpolate.interp1d + """ + if not x: + return var.copy() + + if len(x) == 1: + func, kwargs = _get_interpolator(method, vectorizeable_only=True, + **kwargs) + else: + func, kwargs = _get_interpolator_nd(method, **kwargs) + + if isinstance(var, dask_array_type): + import dask.array as da + + _assert_single_chunk(var, range(var.ndim - len(x), var.ndim)) + chunks = var.chunks[:-len(x)] + new_x[0].shape + drop_axis = range(var.ndim - len(x), var.ndim) + new_axis = range(var.ndim - len(x), var.ndim - len(x) + new_x[0].ndim) + return da.map_blocks(_interpnd, var, x, new_x, func, kwargs, + dtype=var.dtype, chunks=chunks, + new_axis=new_axis, drop_axis=drop_axis) + + return _interpnd(var, x, new_x, func, kwargs) + + +def _interp1d(var, x, new_x, func, kwargs): + # x, new_x are tuples of size 1. + x, new_x = x[0], new_x[0] + rslt = func(x, var, assume_sorted=True, **kwargs)(np.ravel(new_x)) + if new_x.ndim > 1: + return rslt.reshape(var.shape[:-1] + new_x.shape) + if new_x.ndim == 0: + return rslt[..., -1] + return rslt + + +def _interpnd(var, x, new_x, func, kwargs): + if len(x) == 1: + return _interp1d(var, x, new_x, func, kwargs) + + # move the interpolation axes to the start position + var = var.transpose(range(-len(x), var.ndim - len(x))) + # stack new_x to 1 vector, with reshape + xi = np.stack([x1.values.ravel() for x1 in new_x], axis=-1) + rslt = func(x, var, xi, **kwargs) + # move back the interpolation axes to the last position + rslt = rslt.transpose(range(-rslt.ndim + 1, 1)) + return rslt.reshape(rslt.shape[:-1] + new_x[0].shape) diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py new file mode 100644 index 00000000000..592854a4d1b --- /dev/null +++ b/xarray/tests/test_interp.py @@ -0,0 +1,432 @@ +from __future__ import absolute_import, division, print_function + +import numpy as np +import pytest + +import xarray as xr +from xarray.tests import assert_allclose, assert_equal, requires_scipy +from . import has_dask, has_scipy +from .test_dataset import create_test_data + +try: + import scipy +except ImportError: + pass + + +def get_example_data(case): + x = np.linspace(0, 1, 100) + y = np.linspace(0, 0.1, 30) + data = xr.DataArray( + np.sin(x[:, np.newaxis]) * np.cos(y), dims=['x', 'y'], + coords={'x': x, 'y': y, 'x2': ('x', x**2)}) + + if case == 0: + return data + elif case == 1: + return data.chunk({'y': 3}) + elif case == 2: + return data.chunk({'x': 25, 'y': 3}) + elif case == 3: + x = np.linspace(0, 1, 100) + y = np.linspace(0, 0.1, 30) + z = np.linspace(0.1, 0.2, 10) + return xr.DataArray( + np.sin(x[:, np.newaxis, np.newaxis]) * np.cos( + y[:, np.newaxis]) * z, + dims=['x', 'y', 'z'], + coords={'x': x, 'y': y, 'x2': ('x', x**2), 'z': z}) + elif case == 4: + return get_example_data(3).chunk({'z': 5}) + + +def test_keywargs(): + if not has_scipy: + pytest.skip('scipy is not installed.') + + da = get_example_data(0) + assert_equal(da.interp(x=[0.5, 0.8]), da.interp({'x': [0.5, 0.8]})) + + +@pytest.mark.parametrize('method', ['linear', 'cubic']) +@pytest.mark.parametrize('dim', ['x', 'y']) +@pytest.mark.parametrize('case', [0, 1]) +def test_interpolate_1d(method, dim, case): + if not has_scipy: + pytest.skip('scipy is not installed.') + + if not has_dask and case in [1]: + pytest.skip('dask is not installed in the environment.') + + da = get_example_data(case) + xdest = np.linspace(0.0, 0.9, 80) + + if dim == 'y' and case == 1: + with pytest.raises(NotImplementedError): + actual = da.interp(method=method, **{dim: xdest}) + pytest.skip('interpolation along chunked dimension is ' + 'not yet supported') + + actual = da.interp(method=method, **{dim: xdest}) + + # scipy interpolation for the reference + def func(obj, new_x): + return scipy.interpolate.interp1d( + da[dim], obj.data, axis=obj.get_axis_num(dim), bounds_error=False, + fill_value=np.nan, kind=method)(new_x) + + if dim == 'x': + coords = {'x': xdest, 'y': da['y'], 'x2': ('x', func(da['x2'], xdest))} + else: # y + coords = {'x': da['x'], 'y': xdest, 'x2': da['x2']} + + expected = xr.DataArray(func(da, xdest), dims=['x', 'y'], coords=coords) + assert_allclose(actual, expected) + + +@pytest.mark.parametrize('method', ['cubic', 'zero']) +def test_interpolate_1d_methods(method): + if not has_scipy: + pytest.skip('scipy is not installed.') + + da = get_example_data(0) + dim = 'x' + xdest = np.linspace(0.0, 0.9, 80) + + actual = da.interp(method=method, **{dim: xdest}) + + # scipy interpolation for the reference + def func(obj, new_x): + return scipy.interpolate.interp1d( + da[dim], obj.data, axis=obj.get_axis_num(dim), bounds_error=False, + fill_value=np.nan, kind=method)(new_x) + + coords = {'x': xdest, 'y': da['y'], 'x2': ('x', func(da['x2'], xdest))} + expected = xr.DataArray(func(da, xdest), dims=['x', 'y'], coords=coords) + assert_allclose(actual, expected) + + +@pytest.mark.parametrize('use_dask', [False, True]) +def test_interpolate_vectorize(use_dask): + if not has_scipy: + pytest.skip('scipy is not installed.') + + if not has_dask and use_dask: + pytest.skip('dask is not installed in the environment.') + + # scipy interpolation for the reference + def func(obj, dim, new_x): + shape = [s for i, s in enumerate(obj.shape) + if i != obj.get_axis_num(dim)] + for s in new_x.shape[::-1]: + shape.insert(obj.get_axis_num(dim), s) + + return scipy.interpolate.interp1d( + da[dim], obj.data, axis=obj.get_axis_num(dim), + bounds_error=False, fill_value=np.nan)(new_x).reshape(shape) + + da = get_example_data(0) + if use_dask: + da = da.chunk({'y': 5}) + + # xdest is 1d but has different dimension + xdest = xr.DataArray(np.linspace(0.1, 0.9, 30), dims='z', + coords={'z': np.random.randn(30), + 'z2': ('z', np.random.randn(30))}) + + actual = da.interp(x=xdest, method='linear') + + expected = xr.DataArray(func(da, 'x', xdest), dims=['z', 'y'], + coords={'z': xdest['z'], 'z2': xdest['z2'], + 'y': da['y'], + 'x': ('z', xdest.values), + 'x2': ('z', func(da['x2'], 'x', xdest))}) + assert_allclose(actual, expected.transpose('z', 'y')) + + # xdest is 2d + xdest = xr.DataArray(np.linspace(0.1, 0.9, 30).reshape(6, 5), + dims=['z', 'w'], + coords={'z': np.random.randn(6), + 'w': np.random.randn(5), + 'z2': ('z', np.random.randn(6))}) + + actual = da.interp(x=xdest, method='linear') + + expected = xr.DataArray( + func(da, 'x', xdest), + dims=['z', 'w', 'y'], + coords={'z': xdest['z'], 'w': xdest['w'], 'z2': xdest['z2'], + 'y': da['y'], 'x': (('z', 'w'), xdest), + 'x2': (('z', 'w'), func(da['x2'], 'x', xdest))}) + assert_allclose(actual, expected.transpose('z', 'w', 'y')) + + +@pytest.mark.parametrize('case', [3, 4]) +def test_interpolate_nd(case): + if not has_scipy: + pytest.skip('scipy is not installed.') + + if not has_dask and case == 4: + pytest.skip('dask is not installed in the environment.') + + da = get_example_data(case) + + # grid -> grid + xdest = np.linspace(0.1, 1.0, 11) + ydest = np.linspace(0.0, 0.2, 10) + actual = da.interp(x=xdest, y=ydest, method='linear') + + # linear interpolation is separateable + expected = da.interp(x=xdest, method='linear') + expected = expected.interp(y=ydest, method='linear') + assert_allclose(actual.transpose('x', 'y', 'z'), + expected.transpose('x', 'y', 'z')) + + # grid -> 1d-sample + xdest = xr.DataArray(np.linspace(0.1, 1.0, 11), dims='y') + ydest = xr.DataArray(np.linspace(0.0, 0.2, 11), dims='y') + actual = da.interp(x=xdest, y=ydest, method='linear') + + # linear interpolation is separateable + expected_data = scipy.interpolate.RegularGridInterpolator( + (da['x'], da['y']), da.transpose('x', 'y', 'z').values, + method='linear', bounds_error=False, + fill_value=np.nan)(np.stack([xdest, ydest], axis=-1)) + expected = xr.DataArray( + expected_data, dims=['y', 'z'], + coords={'z': da['z'], 'y': ydest, 'x': ('y', xdest.values), + 'x2': da['x2'].interp(x=xdest)}) + assert_allclose(actual.transpose('y', 'z'), expected) + + # reversed order + actual = da.interp(y=ydest, x=xdest, method='linear') + assert_allclose(actual.transpose('y', 'z'), expected) + + +@pytest.mark.parametrize('method', ['linear']) +@pytest.mark.parametrize('case', [0, 1]) +def test_interpolate_scalar(method, case): + if not has_scipy: + pytest.skip('scipy is not installed.') + + if not has_dask and case in [1]: + pytest.skip('dask is not installed in the environment.') + + da = get_example_data(case) + xdest = 0.4 + + actual = da.interp(x=xdest, method=method) + + # scipy interpolation for the reference + def func(obj, new_x): + return scipy.interpolate.interp1d( + da['x'], obj.data, axis=obj.get_axis_num('x'), bounds_error=False, + fill_value=np.nan)(new_x) + + coords = {'x': xdest, 'y': da['y'], 'x2': func(da['x2'], xdest)} + expected = xr.DataArray(func(da, xdest), dims=['y'], coords=coords) + assert_allclose(actual, expected) + + +@pytest.mark.parametrize('method', ['linear']) +@pytest.mark.parametrize('case', [3, 4]) +def test_interpolate_nd_scalar(method, case): + if not has_scipy: + pytest.skip('scipy is not installed.') + + if not has_dask and case in [4]: + pytest.skip('dask is not installed in the environment.') + + da = get_example_data(case) + xdest = 0.4 + ydest = 0.05 + + actual = da.interp(x=xdest, y=ydest, method=method) + # scipy interpolation for the reference + expected_data = scipy.interpolate.RegularGridInterpolator( + (da['x'], da['y']), da.transpose('x', 'y', 'z').values, + method='linear', bounds_error=False, + fill_value=np.nan)(np.stack([xdest, ydest], axis=-1)) + + coords = {'x': xdest, 'y': ydest, 'x2': da['x2'].interp(x=xdest), + 'z': da['z']} + expected = xr.DataArray(expected_data[0], dims=['z'], coords=coords) + assert_allclose(actual, expected) + + +@pytest.mark.parametrize('use_dask', [True, False]) +def test_nans(use_dask): + if not has_scipy: + pytest.skip('scipy is not installed.') + + da = xr.DataArray([0, 1, np.nan, 2], dims='x', coords={'x': range(4)}) + + if not has_dask and use_dask: + pytest.skip('dask is not installed in the environment.') + da = da.chunk() + + actual = da.interp(x=[0.5, 1.5]) + # not all values are nan + assert actual.count() > 0 + + +@pytest.mark.parametrize('use_dask', [True, False]) +def test_errors(use_dask): + if not has_scipy: + pytest.skip('scipy is not installed.') + + # akima and spline are unavailable + da = xr.DataArray([0, 1, np.nan, 2], dims='x', coords={'x': range(4)}) + if not has_dask and use_dask: + pytest.skip('dask is not installed in the environment.') + da = da.chunk() + + for method in ['akima', 'spline']: + with pytest.raises(ValueError): + da.interp(x=[0.5, 1.5], method=method) + + # not sorted + if use_dask: + da = get_example_data(3) + else: + da = get_example_data(1) + + result = da.interp(x=[-1, 1, 3], kwargs={'fill_value': 0.0}) + assert not np.isnan(result.values).any() + result = da.interp(x=[-1, 1, 3]) + assert np.isnan(result.values).any() + + # invalid method + with pytest.raises(ValueError): + da.interp(x=[2, 0], method='boo') + with pytest.raises(ValueError): + da.interp(x=[2, 0], y=2, method='cubic') + with pytest.raises(ValueError): + da.interp(y=[2, 0], method='boo') + + # object-type DataArray cannot be interpolated + da = xr.DataArray(['a', 'b', 'c'], dims='x', coords={'x': [0, 1, 2]}) + with pytest.raises(TypeError): + da.interp(x=0) + + +@requires_scipy +def test_dtype(): + ds = xr.Dataset({'var1': ('x', [0, 1, 2]), 'var2': ('x', ['a', 'b', 'c'])}, + coords={'x': [0.1, 0.2, 0.3], 'z': ('x', ['a', 'b', 'c'])}) + actual = ds.interp(x=[0.15, 0.25]) + assert 'var1' in actual + assert 'var2' not in actual + # object array should be dropped + assert 'z' not in actual.coords + + +@requires_scipy +def test_sorted(): + # unsorted non-uniform gridded data + x = np.random.randn(100) + y = np.random.randn(30) + z = np.linspace(0.1, 0.2, 10) * 3.0 + da = xr.DataArray( + np.cos(x[:, np.newaxis, np.newaxis]) * np.cos( + y[:, np.newaxis]) * z, + dims=['x', 'y', 'z'], + coords={'x': x, 'y': y, 'x2': ('x', x**2), 'z': z}) + + x_new = np.linspace(0, 1, 30) + y_new = np.linspace(0, 1, 20) + + da_sorted = da.sortby('x') + assert_allclose(da.interp(x=x_new), + da_sorted.interp(x=x_new, assume_sorted=True)) + da_sorted = da.sortby(['x', 'y']) + assert_allclose(da.interp(x=x_new, y=y_new), + da_sorted.interp(x=x_new, y=y_new, assume_sorted=True)) + + with pytest.raises(ValueError): + da.interp(x=[0, 1, 2], assume_sorted=True) + + +@requires_scipy +def test_dimension_wo_coords(): + da = xr.DataArray(np.arange(12).reshape(3, 4), dims=['x', 'y'], + coords={'y': [0, 1, 2, 3]}) + da_w_coord = da.copy() + da_w_coord['x'] = np.arange(3) + + assert_equal(da.interp(x=[0.1, 0.2, 0.3]), + da_w_coord.interp(x=[0.1, 0.2, 0.3])) + assert_equal(da.interp(x=[0.1, 0.2, 0.3], y=[0.5]), + da_w_coord.interp(x=[0.1, 0.2, 0.3], y=[0.5])) + + +@requires_scipy +def test_dataset(): + ds = create_test_data() + ds.attrs['foo'] = 'var' + ds['var1'].attrs['buz'] = 'var2' + new_dim2 = xr.DataArray([0.11, 0.21, 0.31], dims='z') + interpolated = ds.interp(dim2=new_dim2) + + assert_allclose(interpolated['var1'], ds['var1'].interp(dim2=new_dim2)) + assert interpolated['var3'].equals(ds['var3']) + + # make sure modifying interpolated does not affect the original dataset + interpolated['var1'][:, 1] = 1.0 + interpolated['var2'][:, 1] = 1.0 + interpolated['var3'][:, 1] = 1.0 + + assert not interpolated['var1'].equals(ds['var1']) + assert not interpolated['var2'].equals(ds['var2']) + assert not interpolated['var3'].equals(ds['var3']) + # attrs should be kept + assert interpolated.attrs['foo'] == 'var' + assert interpolated['var1'].attrs['buz'] == 'var2' + + +@pytest.mark.parametrize('case', [0, 3]) +def test_interpolate_dimorder(case): + """ Make sure the resultant dimension order is consistent with .sel() """ + if not has_scipy: + pytest.skip('scipy is not installed.') + + da = get_example_data(case) + + new_x = xr.DataArray([0, 1, 2], dims='x') + assert da.interp(x=new_x).dims == da.sel(x=new_x, method='nearest').dims + + new_y = xr.DataArray([0, 1, 2], dims='y') + actual = da.interp(x=new_x, y=new_y).dims + expected = da.sel(x=new_x, y=new_y, method='nearest').dims + assert actual == expected + # reversed order + actual = da.interp(y=new_y, x=new_x).dims + expected = da.sel(y=new_y, x=new_x, method='nearest').dims + assert actual == expected + + new_x = xr.DataArray([0, 1, 2], dims='a') + assert da.interp(x=new_x).dims == da.sel(x=new_x, method='nearest').dims + assert da.interp(y=new_x).dims == da.sel(y=new_x, method='nearest').dims + new_y = xr.DataArray([0, 1, 2], dims='a') + actual = da.interp(x=new_x, y=new_y).dims + expected = da.sel(x=new_x, y=new_y, method='nearest').dims + assert actual == expected + + new_x = xr.DataArray([[0], [1], [2]], dims=['a', 'b']) + assert da.interp(x=new_x).dims == da.sel(x=new_x, method='nearest').dims + assert da.interp(y=new_x).dims == da.sel(y=new_x, method='nearest').dims + + if case == 3: + new_x = xr.DataArray([[0], [1], [2]], dims=['a', 'b']) + new_z = xr.DataArray([[0], [1], [2]], dims=['a', 'b']) + actual = da.interp(x=new_x, z=new_z).dims + expected = da.sel(x=new_x, z=new_z, method='nearest').dims + assert actual == expected + + actual = da.interp(z=new_z, x=new_x).dims + expected = da.sel(z=new_z, x=new_x, method='nearest').dims + assert actual == expected + + actual = da.interp(x=0.5, z=new_z).dims + expected = da.sel(x=0.5, z=new_z, method='nearest').dims + assert actual == expected From 98e6a4b84dd2cf4296a3e0aa9710bb79411354e4 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Fri, 8 Jun 2018 10:31:18 +0900 Subject: [PATCH 147/282] reduce memory consumption. (#2220) --- doc/interpolation.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/interpolation.rst b/doc/interpolation.rst index c5fd5166aeb..98cd89afbd4 100644 --- a/doc/interpolation.rst +++ b/doc/interpolation.rst @@ -219,16 +219,16 @@ Let's see how :py:meth:`~xarray.DataArray.interp` works on real data. .. ipython:: python # Raw data - ds = xr.tutorial.load_dataset('air_temperature') + ds = xr.tutorial.load_dataset('air_temperature').isel(time=0) fig, axes = plt.subplots(ncols=2, figsize=(10, 4)) - ds.air.isel(time=0).plot(ax=axes[0]) + ds.air.plot(ax=axes[0]) axes[0].set_title('Raw data') # Interpolated data new_lon = np.linspace(ds.lon[0], ds.lon[-1], ds.dims['lon'] * 4) new_lat = np.linspace(ds.lat[0], ds.lat[-1], ds.dims['lat'] * 4) dsi = ds.interp(lat=new_lat, lon=new_lon) - dsi.air.isel(time=0).plot(ax=axes[1]) + dsi.air.plot(ax=axes[1]) @savefig interpolation_sample3.png width=8in axes[1].set_title('Interpolated data') @@ -247,7 +247,7 @@ The remapping can be done as follows dims=['x', 'z'], coords={'x': x, 'z': z}) fig, axes = plt.subplots(ncols=2, figsize=(10, 4)) - ds.air.isel(time=0).plot(ax=axes[0]) + ds.air.plot(ax=axes[0]) # draw the new coordinate on the original coordinates. for idx in [0, 33, 66, 99]: axes[0].plot(lon.isel(x=idx), lat, '--k') @@ -256,6 +256,6 @@ The remapping can be done as follows axes[0].set_title('Raw data') dsi = ds.interp(lon=lon, lat=lat) - dsi.air.isel(time=0).plot(ax=axes[1]) + dsi.air.plot(ax=axes[1]) @savefig interpolation_sample4.png width=8in axes[1].set_title('Remapped data') From 43c189806264e15cbcae9a37d6f22e2b3e609348 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 7 Jun 2018 21:08:53 -0700 Subject: [PATCH 148/282] DOC: misc fixes to whats-new for 0.10.7 (#2221) --- doc/whats-new.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 44f829874ac..94c3164247f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -35,18 +35,21 @@ Documentation Enhancements ~~~~~~~~~~~~ -- Plot labels now make use of metadata that follow CF conventions. + +- Plot labels now make use of metadata that follow CF conventions + (:issue:`2135`). By `Deepak Cherian `_ and `Ryan Abernathey `_. +- Line plots now support facetting with ``row`` and ``col`` arguments + (:issue:`2107`). + By `Yohai Bar Sinai `_. + - :py:meth:`~xarray.DataArray.interp` and :py:meth:`~xarray.Dataset.interp` methods are newly added. See :ref:`interpolating values with interp` for the detail. (:issue:`2079`) By `Keisuke Fujii `_. -- `:py:meth:`~DataArray.dot` and :py:func:`~dot` are partly supported with older - dask<0.17.4. (related to :issue:`2203`) - By `Keisuke Fujii Date: Thu, 7 Jun 2018 21:35:13 -0700 Subject: [PATCH 149/282] Release v0.10.7 --- doc/whats-new.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 94c3164247f..4e5df777e3a 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -27,11 +27,8 @@ What's New .. _whats-new.0.10.7: -v0.10.7 (unreleased) --------------------- - -Documentation -~~~~~~~~~~~~~ +v0.10.7 (7 June 2018) +--------------------- Enhancements ~~~~~~~~~~~~ From 6c3abedf906482111b06207b9016ea8493c42713 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 7 Jun 2018 21:38:07 -0700 Subject: [PATCH 150/282] Revert to dev version for v0.10.8 --- doc/whats-new.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4e5df777e3a..0dc92fbce58 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,20 @@ What's New - `Python 3 Statement `__ - `Tips on porting to Python 3 `__ +.. _whats-new.0.10.8: + +v0.10.8 (unreleased) +-------------------- + +Documentation +~~~~~~~~~~~~~ + +Enhancements +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + .. _whats-new.0.10.7: v0.10.7 (7 June 2018) From 66be9c5db7d86ea385c3a4cd4295bfce67e3f25b Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 12 Jun 2018 22:51:35 -0700 Subject: [PATCH 151/282] fix zarr chunking bug (#2228) --- doc/whats-new.rst | 7 ++++++- xarray/backends/zarr.py | 30 ++++++++++++------------------ xarray/tests/test_backends.py | 11 +++++++++++ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0dc92fbce58..5871b8bb0a3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -39,6 +39,10 @@ Enhancements Bug fixes ~~~~~~~~~ +- Fixed a bug in ``zarr`` backend which prevented use with datasets with + incomplete chunks in multiple dimensions (:issue:`2225`). + By `Joe Hamman `_. + .. _whats-new.0.10.7: v0.10.7 (7 June 2018) @@ -60,12 +64,13 @@ Enhancements See :ref:`interpolating values with interp` for the detail. (:issue:`2079`) By `Keisuke Fujii `_. - + Bug fixes ~~~~~~~~~ - Fixed a bug in ``rasterio`` backend which prevented use with ``distributed``. The ``rasterio`` backend now returns pickleable objects (:issue:`2021`). + By `Joe Hamman `_. .. _whats-new.0.10.6: diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 343690eaabd..c5043ce8a47 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -78,24 +78,18 @@ def _determine_zarr_chunks(enc_chunks, var_chunks, ndim): # while dask chunks can be variable sized # http://dask.pydata.org/en/latest/array-design.html#chunks if var_chunks and enc_chunks is None: - all_var_chunks = list(product(*var_chunks)) - first_var_chunk = all_var_chunks[0] - # all but the last chunk have to match exactly - for this_chunk in all_var_chunks[:-1]: - if this_chunk != first_var_chunk: - raise ValueError( - "Zarr requires uniform chunk sizes excpet for final chunk." - " Variable %r has incompatible chunks. Consider " - "rechunking using `chunk()`." % (var_chunks,)) - # last chunk is allowed to be smaller - last_var_chunk = all_var_chunks[-1] - for len_first, len_last in zip(first_var_chunk, last_var_chunk): - if len_last > len_first: - raise ValueError( - "Final chunk of Zarr array must be smaller than first. " - "Variable %r has incompatible chunks. Consider rechunking " - "using `chunk()`." % var_chunks) - return first_var_chunk + if any(len(set(chunks[:-1])) > 1 for chunks in var_chunks): + raise ValueError( + "Zarr requires uniform chunk sizes excpet for final chunk." + " Variable %r has incompatible chunks. Consider " + "rechunking using `chunk()`." % (var_chunks,)) + if any((chunks[0] < chunks[-1]) for chunks in var_chunks): + raise ValueError( + "Final chunk of Zarr array must be smaller than first. " + "Variable %r has incompatible chunks. Consider rechunking " + "using `chunk()`." % var_chunks) + # return the first chunk for each dimension + return tuple(chunk[0] for chunk in var_chunks) # from here on, we are dealing with user-specified chunks in encoding # zarr allows chunks to be an integer, in which case it uses the same chunk diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index df7ed66f4fd..e83b80a6dd8 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1330,6 +1330,17 @@ def test_auto_chunk(self): # chunk size should be the same as original self.assertEqual(v.chunks, original[k].chunks) + def test_write_uneven_dask_chunks(self): + # regression for GH#2225 + original = create_test_data().chunk({'dim1': 3, 'dim2': 4, 'dim3': 3}) + + with self.roundtrip( + original, open_kwargs={'auto_chunk': True}) as actual: + for k, v in actual.data_vars.items(): + print(k) + assert v.chunks == actual[k].chunks + + def test_chunk_encoding(self): # These datasets have no dask chunks. All chunking specified in # encoding From 59ad782f29a0f4766bac7802be6650be61f018b8 Mon Sep 17 00:00:00 2001 From: Keisuke Fujii Date: Wed, 20 Jun 2018 10:39:23 +0900 Subject: [PATCH 152/282] implement interp_like (#2222) * implement interp_like * flake8 * interp along datetime * Support datetime coordinate * Using reindex for object coordinate. * Update based on the comments --- doc/api.rst | 2 ++ doc/indexing.rst | 8 ++++++ doc/interpolation.rst | 27 +++++++++++++++++- doc/whats-new.rst | 6 ++++ xarray/core/dataarray.py | 48 +++++++++++++++++++++++++++++++ xarray/core/dataset.py | 57 +++++++++++++++++++++++++++++++++++++ xarray/core/missing.py | 22 ++++++++++++++ xarray/tests/test_interp.py | 48 +++++++++++++++++++++++++++++++ 8 files changed, 217 insertions(+), 1 deletion(-) diff --git a/doc/api.rst b/doc/api.rst index cb44ef82c8f..927c0aa072c 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -111,6 +111,7 @@ Indexing Dataset.sel Dataset.squeeze Dataset.interp + Dataset.interp_like Dataset.reindex Dataset.reindex_like Dataset.set_index @@ -265,6 +266,7 @@ Indexing DataArray.sel DataArray.squeeze DataArray.interp + DataArray.interp_like DataArray.reindex DataArray.reindex_like DataArray.set_index diff --git a/doc/indexing.rst b/doc/indexing.rst index a44e64e4079..c05bf9994fc 100644 --- a/doc/indexing.rst +++ b/doc/indexing.rst @@ -193,6 +193,14 @@ Indexing axes with monotonic decreasing labels also works, as long as the reversed_da.loc[3.1:0.9] +.. note:: + + If you want to interpolate along coordinates rather than looking up the + nearest neighbors, use :py:meth:`~xarray.Dataset.interp` and + :py:meth:`~xarray.Dataset.interp_like`. + See :ref:`interpolation ` for the details. + + Dataset indexing ---------------- diff --git a/doc/interpolation.rst b/doc/interpolation.rst index 98cd89afbd4..cd1c078fb2d 100644 --- a/doc/interpolation.rst +++ b/doc/interpolation.rst @@ -34,7 +34,7 @@ indexing of a :py:class:`~xarray.DataArray`, da.sel(time=3) # interpolation - da.interp(time=3.5) + da.interp(time=2.5) Similar to the indexing, :py:meth:`~xarray.DataArray.interp` also accepts an @@ -82,6 +82,31 @@ Array-like coordinates are also accepted: da.interp(time=[1.5, 2.5], space=[0.15, 0.25]) +:py:meth:`~xarray.DataArray.interp_like` method is a useful shortcut. This +method interpolates an xarray object onto the coordinates of another xarray +object. For example, if we want to compute the difference between +two :py:class:`~xarray.DataArray` s (``da`` and ``other``) staying on slightly +different coordinates, + +.. ipython:: python + + other = xr.DataArray(np.sin(0.4 * np.arange(9).reshape(3, 3)), + [('time', [0.9, 1.9, 2.9]), + ('space', [0.15, 0.25, 0.35])]) + +it might be a good idea to first interpolate ``da`` so that it will stay on the +same coordinates of ``other``, and then subtract it. +:py:meth:`~xarray.DataArray.interp_like` can be used for such a case, + +.. ipython:: python + + # interpolate da along other's coordinates + interpolated = da.interp_like(other) + interpolated + +It is now possible to safely compute the difference ``other - interpolated``. + + Interpolation methods --------------------- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 5871b8bb0a3..55bd0d974f4 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,12 @@ Documentation Enhancements ~~~~~~~~~~~~ +- :py:meth:`~xarray.DataArray.interp_like` and + :py:meth:`~xarray.Dataset.interp_like` methods are newly added. + (:issue:`2218`) + By `Keisuke Fujii `_. + + Bug fixes ~~~~~~~~~ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 4129a3c5f26..35def72c64a 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -951,6 +951,54 @@ def interp(self, coords=None, method='linear', assume_sorted=False, **coords_kwargs) return self._from_temp_dataset(ds) + def interp_like(self, other, method='linear', assume_sorted=False, + kwargs={}): + """Interpolate this object onto the coordinates of another object, + filling out of range values with NaN. + + Parameters + ---------- + other : Dataset or DataArray + Object with an 'indexes' attribute giving a mapping from dimension + names to an 1d array-like, which provides coordinates upon + which to index the variables in this dataset. + method: string, optional. + {'linear', 'nearest'} for multidimensional array, + {'linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic'} + for 1-dimensional array. 'linear' is used by default. + assume_sorted: boolean, optional + If False, values of coordinates that are interpolated over can be + in any order and they are sorted first. If True, interpolated + coordinates are assumed to be an array of monotonically increasing + values. + kwargs: dictionary, optional + Additional keyword passed to scipy's interpolator. + + Returns + ------- + interpolated: xr.DataArray + Another dataarray by interpolating this dataarray's data along the + coordinates of the other object. + + Note + ---- + scipy is required. + If the dataarray has object-type coordinates, reindex is used for these + coordinates instead of the interpolation. + + See Also + -------- + DataArray.interp + DataArray.reindex_like + """ + if self.dtype.kind not in 'uifc': + raise TypeError('interp only works for a numeric type array. ' + 'Given {}.'.format(self.dtype)) + + ds = self._to_temp_dataset().interp_like( + other, method=method, kwargs=kwargs, assume_sorted=assume_sorted) + return self._from_temp_dataset(ds) + def rename(self, new_name_or_name_dict=None, **names): """Returns a new DataArray with renamed coordinates or a new name. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 90712c953da..8e039572237 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -1893,6 +1893,63 @@ def maybe_variable(obj, k): .union(coord_vars)) return obj._replace_vars_and_dims(variables, coord_names=coord_names) + def interp_like(self, other, method='linear', assume_sorted=False, + kwargs={}): + """Interpolate this object onto the coordinates of another object, + filling the out of range values with NaN. + + Parameters + ---------- + other : Dataset or DataArray + Object with an 'indexes' attribute giving a mapping from dimension + names to an 1d array-like, which provides coordinates upon + which to index the variables in this dataset. + method: string, optional. + {'linear', 'nearest'} for multidimensional array, + {'linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic'} + for 1-dimensional array. 'linear' is used by default. + assume_sorted: boolean, optional + If False, values of coordinates that are interpolated over can be + in any order and they are sorted first. If True, interpolated + coordinates are assumed to be an array of monotonically increasing + values. + kwargs: dictionary, optional + Additional keyword passed to scipy's interpolator. + + Returns + ------- + interpolated: xr.Dataset + Another dataset by interpolating this dataset's data along the + coordinates of the other object. + + Note + ---- + scipy is required. + If the dataset has object-type coordinates, reindex is used for these + coordinates instead of the interpolation. + + See Also + -------- + Dataset.interp + Dataset.reindex_like + """ + coords = alignment.reindex_like_indexers(self, other) + + numeric_coords = OrderedDict() + object_coords = OrderedDict() + for k, v in coords.items(): + if v.dtype.kind in 'uifcMm': + numeric_coords[k] = v + else: + object_coords[k] = v + + ds = self + if object_coords: + # We do not support interpolation along object coordinate. + # reindex instead. + ds = self.reindex(object_coords) + return ds.interp(numeric_coords, method, assume_sorted, kwargs) + def rename(self, name_dict=None, inplace=False, **names): """Returns a new object with renamed variables and dimensions. diff --git a/xarray/core/missing.py b/xarray/core/missing.py index e10f37d58d8..743627bb381 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -395,6 +395,26 @@ def _localize(var, indexes_coords): return var.isel(**indexes), indexes_coords +def _floatize_x(x, new_x): + """ Make x and new_x float. + This is particulary useful for datetime dtype. + x, new_x: tuple of np.ndarray + """ + x = list(x) + new_x = list(new_x) + for i in range(len(x)): + if x[i].dtype.kind in 'Mm': + # Scipy casts coordinates to np.float64, which is not accurate + # enough for datetime64 (uses 64bit integer). + # We assume that the most of the bits are used to represent the + # offset (min(x)) and the variation (x - min(x)) can be + # represented by float. + xmin = np.min(x[i]) + x[i] = (x[i] - xmin).astype(np.float64) + new_x[i] = (new_x[i] - xmin).astype(np.float64) + return x, new_x + + def interp(var, indexes_coords, method, **kwargs): """ Make an interpolation of Variable @@ -523,6 +543,8 @@ def _interp1d(var, x, new_x, func, kwargs): def _interpnd(var, x, new_x, func, kwargs): + x, new_x = _floatize_x(x, new_x) + if len(x) == 1: return _interp1d(var, x, new_x, func, kwargs) diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py index 592854a4d1b..69a4644bc97 100644 --- a/xarray/tests/test_interp.py +++ b/xarray/tests/test_interp.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function import numpy as np +import pandas as pd import pytest import xarray as xr @@ -430,3 +431,50 @@ def test_interpolate_dimorder(case): actual = da.interp(x=0.5, z=new_z).dims expected = da.sel(x=0.5, z=new_z, method='nearest').dims assert actual == expected + + +@requires_scipy +def test_interp_like(): + ds = create_test_data() + ds.attrs['foo'] = 'var' + ds['var1'].attrs['buz'] = 'var2' + + other = xr.DataArray(np.random.randn(3), dims=['dim2'], + coords={'dim2': [0, 1, 2]}) + interpolated = ds.interp_like(other) + + assert_allclose(interpolated['var1'], + ds['var1'].interp(dim2=other['dim2'])) + assert_allclose(interpolated['var1'], + ds['var1'].interp_like(other)) + assert interpolated['var3'].equals(ds['var3']) + + # attrs should be kept + assert interpolated.attrs['foo'] == 'var' + assert interpolated['var1'].attrs['buz'] == 'var2' + + other = xr.DataArray(np.random.randn(3), dims=['dim3'], + coords={'dim3': ['a', 'b', 'c']}) + + actual = ds.interp_like(other) + expected = ds.reindex_like(other) + assert_allclose(actual, expected) + + +@requires_scipy +def test_datetime(): + da = xr.DataArray(np.random.randn(24), dims='time', + coords={'time': pd.date_range('2000-01-01', periods=24)}) + + x_new = pd.date_range('2000-01-02', periods=3) + actual = da.interp(time=x_new) + expected = da.isel(time=[1, 2, 3]) + assert_allclose(actual, expected) + + x_new = np.array([np.datetime64('2000-01-01T12:00'), + np.datetime64('2000-01-02T12:00')]) + actual = da.interp(time=x_new) + assert_allclose(actual.isel(time=0).drop('time'), + 0.5 * (da.isel(time=0) + da.isel(time=1))) + assert_allclose(actual.isel(time=1).drop('time'), + 0.5 * (da.isel(time=1) + da.isel(time=2))) From 73b476e4db6631b2203954dd5b138cb650e4fb8c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 20 Jun 2018 09:26:36 -0700 Subject: [PATCH 153/282] Bugfix for faceting line plots. (#2229) * Bugfix for faceting line plots. * Make add_legend public and usable. --- xarray/plot/facetgrid.py | 16 +++++++++------- xarray/tests/test_plot.py | 6 ++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index 771f0879408..a0d7c4dd5e2 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -293,14 +293,16 @@ def map_dataarray_line(self, x=None, y=None, hue=None, **kwargs): ax=ax, _labels=False, **kwargs) self._mappables.append(mappable) - _, _, _, xlabel, ylabel, huelabel = _infer_line_data( + _, _, hueplt, xlabel, ylabel, huelabel = _infer_line_data( darray=self.data.loc[self.name_dicts.flat[0]], x=x, y=y, hue=hue) + self._hue_var = hueplt + self._hue_label = huelabel self._finalize_grid(xlabel, ylabel) - if add_legend and huelabel: - self.add_line_legend(huelabel) + if add_legend and hueplt is not None and huelabel is not None: + self.add_legend() return self @@ -314,12 +316,12 @@ def _finalize_grid(self, *axlabels): if namedict is None: ax.set_visible(False) - def add_line_legend(self, huelabel): + def add_legend(self, **kwargs): figlegend = self.fig.legend( handles=self._mappables[-1], - labels=list(self.data.coords[huelabel].values), - title=huelabel, - loc="center right") + labels=list(self._hue_var.values), + title=self._hue_label, + loc="center right", **kwargs) # Draw the plot to set the bounding boxes correctly self.fig.draw(self.fig.canvas.get_renderer()) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index cdb515ba92e..15729f25e22 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1529,6 +1529,12 @@ def setUp(self): range(3), ['A', 'B', 'C', 'C++']], name='Cornelius Ortega the 1st') + self.darray.hue.name = 'huename' + self.darray.hue.attrs['units'] = 'hunits' + self.darray.x.attrs['units'] = 'xunits' + self.darray.col.attrs['units'] = 'colunits' + self.darray.row.attrs['units'] = 'rowunits' + def test_facetgrid_shape(self): g = self.darray.plot(row='row', col='col', hue='hue') assert g.axes.shape == (len(self.darray.row), len(self.darray.col)) From 9491318e29b478234e6f96c3547d724504b4a1bb Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Fri, 22 Jun 2018 18:00:05 -0400 Subject: [PATCH 154/282] no mode arg to open_zarr (#2246) --- doc/io.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/io.rst b/doc/io.rst index 7f7e7a2a66a..093ee773e15 100644 --- a/doc/io.rst +++ b/doc/io.rst @@ -603,7 +603,7 @@ pass to xarray:: # write to the bucket ds.to_zarr(store=gcsmap) # read it back - ds_gcs = xr.open_zarr(gcsmap, mode='r') + ds_gcs = xr.open_zarr(gcsmap) .. _Zarr: http://zarr.readthedocs.io/ .. _Amazon S3: https://aws.amazon.com/s3/ From 04a78d50a928f4af2efc4e1d19370c76d822dbb6 Mon Sep 17 00:00:00 2001 From: Mike Neish <1554921+neishm@users.noreply.github.com> Date: Fri, 29 Jun 2018 01:07:26 -0400 Subject: [PATCH 155/282] Write inconsistent chunks to netcdf (#2257) * Test case for writing Datasets to netCDF4 where each DataArray has different chunk sizes. * When writing Datasets to netCDF4, don't need the chunk sizes to be consistent over all arrays. Closes #2254. * Added a note about the bugfix for #2254. --- doc/whats-new.rst | 4 ++++ xarray/backends/api.py | 5 +++-- xarray/tests/test_backends.py | 18 +++++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 55bd0d974f4..322e21acc17 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -49,6 +49,10 @@ Bug fixes incomplete chunks in multiple dimensions (:issue:`2225`). By `Joe Hamman `_. +- Fixed a bug in :py:meth:`~Dataset.to_netcdf` which prevented writing + datasets when the arrays had different chunk sizes (:issue:`2254`). + By `Mike Neish `_. + .. _whats-new.0.10.7: v0.10.7 (7 June 2018) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 753f8394a7b..d5e2e8bbc2c 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -700,13 +700,14 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, # handle scheduler specific logic scheduler = get_scheduler() - if (dataset.chunks and scheduler in ['distributed', 'multiprocessing'] and + have_chunks = any(v.chunks for v in dataset.variables.values()) + if (have_chunks and scheduler in ['distributed', 'multiprocessing'] and engine != 'netcdf4'): raise NotImplementedError("Writing netCDF files with the %s backend " "is not currently supported with dask's %s " "scheduler" % (engine, scheduler)) lock = _get_lock(engine, scheduler, format, path_or_file) - autoclose = (dataset.chunks and + autoclose = (have_chunks and scheduler in ['distributed', 'multiprocessing']) target = path_or_file if path_or_file is not None else BytesIO() diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index e83b80a6dd8..9ec68bb0846 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1275,6 +1275,23 @@ def test_dataset_caching(self): # caching behavior differs for dask pass + def test_write_inconsistent_chunks(self): + # Construct two variables with the same dimensions, but different + # chunk sizes. + x = da.zeros((100, 100), dtype='f4', chunks=(50, 100)) + x = DataArray(data=x, dims=('lat', 'lon'), name='x') + x.encoding['chunksizes'] = (50, 100) + x.encoding['original_shape'] = (100, 100) + y = da.ones((100, 100), dtype='f4', chunks=(100, 50)) + y = DataArray(data=y, dims=('lat', 'lon'), name='y') + y.encoding['chunksizes'] = (100, 50) + y.encoding['original_shape'] = (100, 100) + # Put them both into the same dataset + ds = Dataset({'x': x, 'y': y}) + with self.roundtrip(ds) as actual: + assert actual['x'].encoding['chunksizes'] == (50, 100) + assert actual['y'].encoding['chunksizes'] == (100, 50) + class NetCDF4ViaDaskDataTestAutocloseTrue(NetCDF4ViaDaskDataTest): autoclose = True @@ -1340,7 +1357,6 @@ def test_write_uneven_dask_chunks(self): print(k) assert v.chunks == actual[k].chunks - def test_chunk_encoding(self): # These datasets have no dask chunks. All chunking specified in # encoding From 5ddfb6dc07f9de08bf95232df4a35e95a85fd113 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Tue, 3 Jul 2018 11:16:26 +0200 Subject: [PATCH 156/282] Plotting: do not check for monotonicity with 2D coords (#2260) * Plotting: do not check for monotonicity with 2D coords * Whats new --- doc/whats-new.rst | 4 ++++ xarray/plot/plot.py | 8 ++++---- xarray/tests/test_plot.py | 26 ++++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 322e21acc17..03ce381c516 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -53,6 +53,10 @@ Bug fixes datasets when the arrays had different chunk sizes (:issue:`2254`). By `Mike Neish `_. +- Fixed a bug in 2D plots which incorrectly raised an error when 2D coordinates + weren't monotonic (:issue:`2250`). + By `Fabien Maussion `_. + .. _whats-new.0.10.7: v0.10.7 (7 June 2018) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 6322fc09d92..19b6420f35b 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -872,7 +872,7 @@ def _is_monotonic(coord, axis=0): return np.all(delta_pos) or np.all(delta_neg) -def _infer_interval_breaks(coord, axis=0): +def _infer_interval_breaks(coord, axis=0, check_monotonic=False): """ >>> _infer_interval_breaks(np.arange(5)) array([-0.5, 0.5, 1.5, 2.5, 3.5, 4.5]) @@ -882,7 +882,7 @@ def _infer_interval_breaks(coord, axis=0): """ coord = np.asarray(coord) - if not _is_monotonic(coord, axis=axis): + if check_monotonic and not _is_monotonic(coord, axis=axis): raise ValueError("The input coordinate is not sorted in increasing " "order along axis %d. This can lead to unexpected " "results. Consider calling the `sortby` method on " @@ -921,8 +921,8 @@ def pcolormesh(x, y, z, ax, infer_intervals=None, **kwargs): if infer_intervals: if len(x.shape) == 1: - x = _infer_interval_breaks(x) - y = _infer_interval_breaks(y) + x = _infer_interval_breaks(x, check_monotonic=True) + y = _infer_interval_breaks(y, check_monotonic=True) else: # we have to infer the intervals on both axes x = _infer_interval_breaks(x, axis=1) diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 15729f25e22..4f8a3277f9d 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -228,9 +228,31 @@ def test__infer_interval_breaks(self): np.testing.assert_allclose(xref, x) np.testing.assert_allclose(yref, y) - # test that warning is raised for non-monotonic inputs + # test that ValueError is raised for non-monotonic 1D inputs with pytest.raises(ValueError): - _infer_interval_breaks(np.array([0, 2, 1])) + _infer_interval_breaks(np.array([0, 2, 1]), check_monotonic=True) + + def test_geo_data(self): + # Regression test for gh2250 + # Realistic coordinates taken from the example dataset + lat = np.array([[16.28, 18.48, 19.58, 19.54, 18.35], + [28.07, 30.52, 31.73, 31.68, 30.37], + [39.65, 42.27, 43.56, 43.51, 42.11], + [50.52, 53.22, 54.55, 54.50, 53.06]]) + lon = np.array([[-126.13, -113.69, -100.92, -88.04, -75.29], + [-129.27, -115.62, -101.54, -87.32, -73.26], + [-133.10, -118.00, -102.31, -86.42, -70.76], + [-137.85, -120.99, -103.28, -85.28, -67.62]]) + data = np.sqrt(lon ** 2 + lat ** 2) + da = DataArray(data, dims=('y', 'x'), + coords={'lon': (('y', 'x'), lon), + 'lat': (('y', 'x'), lat)}) + da.plot(x='lon', y='lat') + ax = plt.gca() + assert ax.has_data() + da.plot(x='lat', y='lon') + ax = plt.gca() + assert ax.has_data() def test_datetime_dimension(self): nrow = 3 From a962956486153c644f982704e86ba35c171597aa Mon Sep 17 00:00:00 2001 From: korigod Date: Wed, 4 Jul 2018 19:59:47 +0300 Subject: [PATCH 157/282] doc: corrections of contributing.rst (#2266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * doc: fix typos in contributing.rst `` :issue:`1234` ``, which renders to `` GH1234 ``, is replaced with ``:issue:`1234```, which renders to :issue:`1234`. * doc: fix style in contributing.rst The formatting has been fixed to be more consistent. The lines longer than 90–95 characters have been wrapped where appropriate. * doc: fix path to built htmls in contributing.rst * doc: recommend shorter first lines of commit messages GitHub shows only the first 72 characters of the first line of the commit message as the commit title. If the line is longer, an ugly wrap takes place. --- doc/contributing.rst | 102 +++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 71c952e7e5b..ceba81d9319 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -18,7 +18,7 @@ Where to start? All contributions, bug reports, bug fixes, documentation improvements, enhancements, and ideas are welcome. -If you are brand new to xarray or open-source development, we recommend going +If you are brand new to *xarray* or open-source development, we recommend going through the `GitHub "issues" tab `_ to find issues that interest you. There are a number of issues listed under `Documentation `_ @@ -35,14 +35,14 @@ Feel free to ask questions on the `mailing list Bug reports and enhancement requests ==================================== -Bug reports are an important part of making *xarray* more stable. Having a complete bug report -will allow others to reproduce the bug and provide insight into fixing. See +Bug reports are an important part of making *xarray* more stable. Having a complete bug +report will allow others to reproduce the bug and provide insight into fixing. See `this stackoverflow article `_ for tips on writing a good bug report. Trying the bug-producing code out on the *master* branch is often a worthwhile exercise -to confirm the bug still exists. It is also worth searching existing bug reports and pull requests -to see if the issue has already been reported and/or fixed. +to confirm the bug still exists. It is also worth searching existing bug reports and +pull requests to see if the issue has already been reported and/or fixed. Bug reports must: @@ -56,32 +56,34 @@ Bug reports must: ... ``` -#. Include the full version string of *xarray* and its dependencies. You can use the built in function:: +#. Include the full version string of *xarray* and its dependencies. You can use the + built in function:: >>> import xarray as xr >>> xr.show_versions() #. Explain why the current behavior is wrong/not desired and what you expect instead. -The issue will then show up to the *xarray* community and be open to comments/ideas from others. +The issue will then show up to the *xarray* community and be open to comments/ideas +from others. .. _contributing.github: Working with the code ===================== -Now that you have an issue you want to fix, enhancement to add, or documentation to improve, -you need to learn how to work with GitHub and the *xarray* code base. +Now that you have an issue you want to fix, enhancement to add, or documentation +to improve, you need to learn how to work with GitHub and the *xarray* code base. .. _contributing.version_control: Version control, Git, and GitHub -------------------------------- -To the new user, working with Git is one of the more daunting aspects of contributing to *xarray*. -It can very quickly become overwhelming, but sticking to the guidelines below will help keep the process -straightforward and mostly trouble free. As always, if you are having difficulties please -feel free to ask for help. +To the new user, working with Git is one of the more daunting aspects of contributing +to *xarray*. It can very quickly become overwhelming, but sticking to the guidelines +below will help keep the process straightforward and mostly trouble free. As always, +if you are having difficulties please feel free to ask for help. The code is hosted on `GitHub `_. To contribute you will need to sign up for a `free GitHub account @@ -122,7 +124,7 @@ the upstream (main project) *xarray* repository. Creating a development environment ---------------------------------- -To test out code changes, you'll need to build xarray from source, which +To test out code changes, you'll need to build *xarray* from source, which requires a Python environment. If you're making documentation changes, you can skip to :ref:`contributing.documentation` but you won't be able to build the documentation locally before pushing your changes. @@ -158,7 +160,7 @@ We'll now kick off a two-step process: # Build and install xarray pip install -e . -At this point you should be able to import xarray from your locally built version:: +At this point you should be able to import *xarray* from your locally built version:: $ python # start an interpreter >>> import xarray @@ -201,11 +203,11 @@ To update this branch, you need to retrieve the changes from the master branch:: git fetch upstream git rebase upstream/master -This will replay your commits on top of the latest xarray git master. If this +This will replay your commits on top of the latest *xarray* git master. If this leads to merge conflicts, you must resolve these before submitting your pull -request. If you have uncommitted changes, you will need to ``stash`` them prior -to updating. This will effectively store your changes and they can be reapplied -after updating. +request. If you have uncommitted changes, you will need to ``git stash`` them +prior to updating. This will effectively store your changes and they can be +reapplied after updating. .. _contributing.documentation: @@ -299,7 +301,7 @@ Navigate to your local ``xarray/doc/`` directory in the console and run:: make html -Then you can find the HTML output in the folder ``xarray/doc/build/html/``. +Then you can find the HTML output in the folder ``xarray/doc/_build/html/``. The first time you build the docs, it will take quite a while because it has to run all the code examples and build all the generated docstring pages. In subsequent @@ -361,8 +363,8 @@ Backwards Compatibility ~~~~~~~~~~~~~~~~~~~~~~~ Please try to maintain backward compatibility. *xarray* has growing number of users with -lots of existing code, so don't break it if at all possible. If you think breakage is, -required clearly state why as part of the pull request. Also, be careful when changing +lots of existing code, so don't break it if at all possible. If you think breakage is +required, clearly state why as part of the pull request. Also, be careful when changing method signatures and add deprecation warnings where needed. Also, add the deprecated sphinx directive to the deprecated functions or methods. @@ -379,17 +381,18 @@ services need to be hooked to your GitHub repository. Instructions are here for `Travis-CI `__, and `Appveyor `__. -A pull-request will be considered for merging when you have an all 'green' build. If any tests are failing, -then you will get a red 'X', where you can click through to see the individual failed tests. -This is an example of a green build. +A pull-request will be considered for merging when you have an all 'green' build. If any +tests are failing, then you will get a red 'X', where you can click through to see the +individual failed tests. This is an example of a green build. .. image:: _static/ci.png .. note:: - Each time you push to *your* fork, a *new* run of the tests will be triggered on the CI. Appveyor will auto-cancel - any non-currently-running tests for that same pull-request. You can also enable the auto-cancel feature for - `Travis-CI here `__. + Each time you push to your PR branch, a new run of the tests will be triggered on the CI. + Appveyor will auto-cancel any non-currently-running tests for that same pull-request. + You can also enable the auto-cancel feature for `Travis-CI here + `__. .. _contributing.tdd: @@ -437,7 +440,8 @@ the expected correct result:: Transitioning to ``pytest`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*xarray* existing test structure is *mostly* classed based, meaning that you will typically find tests wrapped in a class. +*xarray* existing test structure is *mostly* classed based, meaning that you will +typically find tests wrapped in a class. .. code-block:: python @@ -460,16 +464,19 @@ Using ``pytest`` Here is an example of a self-contained set of tests that illustrate multiple features that we like to use. -- functional style: tests are like ``test_*`` and *only* take arguments that are either fixtures or parameters +- functional style: tests are like ``test_*`` and *only* take arguments that are either + fixtures or parameters - ``pytest.mark`` can be used to set metadata on test functions, e.g. ``skip`` or ``xfail``. - using ``parametrize``: allow testing of multiple cases - to set a mark on a parameter, ``pytest.param(..., marks=...)`` syntax should be used - ``fixture``, code for object construction, on a per-test basis - using bare ``assert`` for scalars and truth-testing -- ``tm.assert_series_equal`` (and its counter part ``tm.assert_frame_equal``), for xarray object comparisons. +- ``tm.assert_series_equal`` (and its counter part ``tm.assert_frame_equal``), for xarray + object comparisons. - the typical pattern of constructing an ``expected`` and comparing versus the ``result`` -We would name this file ``test_cool_feature.py`` and put in an appropriate place in the ``xarray/tests/`` structure. +We would name this file ``test_cool_feature.py`` and put in an appropriate place in the +``xarray/tests/`` structure. .. TODO: confirm that this actually works @@ -640,9 +647,9 @@ using ``.`` as a separator. For example:: will only run the ``GroupByMethods`` benchmark defined in ``groupby.py``. -You can also run the benchmark suite using the version of ``xarray`` +You can also run the benchmark suite using the version of *xarray* already installed in your current Python environment. This can be -useful if you do not have virtualenv or conda, or are using the +useful if you do not have ``virtualenv`` or ``conda``, or are using the ``setup.py develop`` approach discussed above; for the in-place build you need to set ``PYTHONPATH``, e.g. ``PYTHONPATH="$PWD/.." asv [remaining arguments]``. @@ -661,7 +668,7 @@ This will display stderr from the benchmarks, and use your local Information on how to write a benchmark and how to use asv can be found in the `asv documentation `_. -The ``xarray`` benchmarking suite is run remotely and the results are +The *xarray* benchmarking suite is run remotely and the results are available `here `_. Documenting your code @@ -670,7 +677,7 @@ Documenting your code Changes should be reflected in the release notes located in ``doc/whats-new.rst``. This file contains an ongoing change log for each release. Add an entry to this file to document your fix, enhancement or (unavoidable) breaking change. Make sure to include the -GitHub issue number when adding your entry (using `` :issue:`1234` `` where `1234` is the +GitHub issue number when adding your entry (using ``:issue:`1234```, where ``1234`` is the issue/pull request number). If your code is an enhancement, it is most likely necessary to add usage @@ -704,22 +711,23 @@ Finally, commit your changes to your local repository with an explanatory messag *Xarray* uses a convention for commit message prefixes and layout. Here are some common prefixes along with general guidelines for when to use them: - * ENH: Enhancement, new functionality - * BUG: Bug fix - * DOC: Additions/updates to documentation - * TST: Additions/updates to tests - * BLD: Updates to the build process/scripts - * PERF: Performance improvement - * CLN: Code cleanup + * ``ENH``: Enhancement, new functionality + * ``BUG``: Bug fix + * ``DOC``: Additions/updates to documentation + * ``TST``: Additions/updates to tests + * ``BLD``: Updates to the build process/scripts + * ``PERF``: Performance improvement + * ``CLN``: Code cleanup -The following defines how a commit message should be structured. Please reference the -relevant GitHub issues in your commit message using GH1234 or #1234. Either style -is fine, but the former is generally preferred: +The following defines how a commit message should be structured: - * a subject line with `< 80` chars. + * A subject line with `< 72` chars. * One blank line. * Optionally, a commit message body. +Please reference the relevant GitHub issues in your commit message using ``GH1234`` or +``#1234``. Either style is fine, but the former is generally preferred. + Now you can commit your changes in your local repository:: git commit -m From 4173f7bde816caf9a0248774fbcc7a5de13c0b9f Mon Sep 17 00:00:00 2001 From: Stephane Raynaud Date: Wed, 4 Jul 2018 17:01:52 +0000 Subject: [PATCH 158/282] Add curvilinear grid support to to_cdms2 and fix mask bug (#2262) * Add curvilinear grid support to to_cdms2 and fix mask bug * fix indentation in to_cdms2 * Add generic unstructured grid support to _from_cdms2 and to_cdms2 * Fix indentation in from_cdms2 * Fix indentation in from_cdms2 * Split cdms2 unit tests and use OrderedDict --- doc/whats-new.rst | 10 ++++++ xarray/convert.py | 61 ++++++++++++++++++++++++++++++---- xarray/tests/test_dataarray.py | 57 ++++++++++++++++++++++++++++++- 3 files changed, 121 insertions(+), 7 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 03ce381c516..bd23386a460 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -41,6 +41,11 @@ Enhancements (:issue:`2218`) By `Keisuke Fujii `_. +- Added support for curvilinear and unstructured generic grids + to :py:meth:`~xarray.DataArray.to_cdms2` and + :py:meth:`~xarray.DataArray.from_cdms2` (:issue:`2262`). + By `Stephane Raynaud `_. + Bug fixes ~~~~~~~~~ @@ -53,10 +58,15 @@ Bug fixes datasets when the arrays had different chunk sizes (:issue:`2254`). By `Mike Neish `_. +- Fixed masking during the conversion to cdms2 objects by + :py:meth:`~xarray.DataArray.to_cdms2` (:issue:`2262`). + By `Stephane Raynaud `_. + - Fixed a bug in 2D plots which incorrectly raised an error when 2D coordinates weren't monotonic (:issue:`2250`). By `Fabien Maussion `_. + .. _whats-new.0.10.7: v0.10.7 (7 June 2018) diff --git a/xarray/convert.py b/xarray/convert.py index a3c99119306..f3a5ccb2ce5 100644 --- a/xarray/convert.py +++ b/xarray/convert.py @@ -2,7 +2,9 @@ """ from __future__ import absolute_import, division, print_function +from collections import OrderedDict import numpy as np +import pandas as pd from .coding.times import CFDatetimeCoder, CFTimedeltaCoder from .conventions import decode_cf @@ -38,15 +40,28 @@ def from_cdms2(variable): """ values = np.asarray(variable) name = variable.id - coords = [(v.id, np.asarray(v), - _filter_attrs(v.attributes, cdms2_ignored_attrs)) - for v in variable.getAxisList()] + dims = variable.getAxisIds() + coords = {} + for axis in variable.getAxisList(): + coords[axis.id] = DataArray( + np.asarray(axis), dims=[axis.id], + attrs=_filter_attrs(axis.attributes, cdms2_ignored_attrs)) + grid = variable.getGrid() + if grid is not None: + ids = [a.id for a in grid.getAxisList()] + for axis in grid.getLongitude(), grid.getLatitude(): + if axis.id not in variable.getAxisIds(): + coords[axis.id] = DataArray( + np.asarray(axis[:]), dims=ids, + attrs=_filter_attrs(axis.attributes, + cdms2_ignored_attrs)) attrs = _filter_attrs(variable.attributes, cdms2_ignored_attrs) - dataarray = DataArray(values, coords=coords, name=name, attrs=attrs) + dataarray = DataArray(values, dims=dims, coords=coords, name=name, + attrs=attrs) return decode_cf(dataarray.to_dataset())[dataarray.name] -def to_cdms2(dataarray): +def to_cdms2(dataarray, copy=True): """Convert a DataArray into a cdms2 variable """ # we don't want cdms2 to be a hard dependency @@ -56,6 +71,7 @@ def set_cdms2_attrs(var, attrs): for k, v in attrs.items(): setattr(var, k, v) + # 1D axes axes = [] for dim in dataarray.dims: coord = encode(dataarray.coords[dim]) @@ -63,9 +79,42 @@ def set_cdms2_attrs(var, attrs): set_cdms2_attrs(axis, coord.attrs) axes.append(axis) + # Data var = encode(dataarray) - cdms2_var = cdms2.createVariable(var.values, axes=axes, id=dataarray.name) + cdms2_var = cdms2.createVariable(var.values, axes=axes, id=dataarray.name, + mask=pd.isnull(var.values), copy=copy) + + # Attributes set_cdms2_attrs(cdms2_var, var.attrs) + + # Curvilinear and unstructured grids + if dataarray.name not in dataarray.coords: + + cdms2_axes = OrderedDict() + for coord_name in set(dataarray.coords.keys()) - set(dataarray.dims): + + coord_array = dataarray.coords[coord_name].to_cdms2() + + cdms2_axis_cls = (cdms2.coord.TransientAxis2D + if coord_array.ndim else + cdms2.auxcoord.TransientAuxAxis1D) + cdms2_axis = cdms2_axis_cls(coord_array) + if cdms2_axis.isLongitude(): + cdms2_axes['lon'] = cdms2_axis + elif cdms2_axis.isLatitude(): + cdms2_axes['lat'] = cdms2_axis + + if 'lon' in cdms2_axes and 'lat' in cdms2_axes: + if len(cdms2_axes['lon'].shape) == 2: + cdms2_grid = cdms2.hgrid.TransientCurveGrid( + cdms2_axes['lat'], cdms2_axes['lon']) + else: + cdms2_grid = cdms2.gengrid.AbstractGenericGrid( + cdms2_axes['lat'], cdms2_axes['lon']) + for axis in cdms2_grid.getAxisList(): + cdms2_var.setAxis(cdms2_var.getAxisIds().index(axis.id), axis) + cdms2_var.setGrid(cdms2_grid) + return cdms2_var diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index d339e6402b6..df09a1e58df 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -13,6 +13,7 @@ import xarray as xr from xarray import ( DataArray, Dataset, IndexVariable, Variable, align, broadcast, set_options) +from xarray.convert import from_cdms2 from xarray.coding.times import CFDatetimeCoder, _import_cftime from xarray.core.common import full_like from xarray.core.pycompat import OrderedDict, iteritems @@ -2914,7 +2915,8 @@ def test_to_masked_array(self): ma = da.to_masked_array() assert len(ma.mask) == N - def test_to_and_from_cdms2(self): + def test_to_and_from_cdms2_classic(self): + """Classic with 1D axes""" pytest.importorskip('cdms2') original = DataArray( @@ -2940,6 +2942,59 @@ def test_to_and_from_cdms2(self): roundtripped = DataArray.from_cdms2(actual) assert_identical(original, roundtripped) + back = from_cdms2(actual) + self.assertItemsEqual(original.dims, back.dims) + self.assertItemsEqual(original.coords.keys(), back.coords.keys()) + for coord_name in original.coords.keys(): + assert_array_equal(original.coords[coord_name], + back.coords[coord_name]) + + def test_to_and_from_cdms2_sgrid(self): + """Curvilinear (structured) grid + + The rectangular grid case is covered by the classic case + """ + pytest.importorskip('cdms2') + + lonlat = np.mgrid[:3, :4] + lon = DataArray(lonlat[1], dims=['y', 'x'], name='lon') + lat = DataArray(lonlat[0], dims=['y', 'x'], name='lat') + x = DataArray(np.arange(lon.shape[1]), dims=['x'], name='x') + y = DataArray(np.arange(lon.shape[0]), dims=['y'], name='y') + original = DataArray(lonlat.sum(axis=0), dims=['y', 'x'], + coords=OrderedDict(x=x, y=y, lon=lon, lat=lat), + name='sst') + actual = original.to_cdms2() + self.assertItemsEqual(actual.getAxisIds(), original.dims) + assert_array_equal(original.coords['lon'], actual.getLongitude()) + assert_array_equal(original.coords['lat'], actual.getLatitude()) + + back = from_cdms2(actual) + self.assertItemsEqual(original.dims, back.dims) + self.assertItemsEqual(original.coords.keys(), back.coords.keys()) + assert_array_equal(original.coords['lat'], back.coords['lat']) + assert_array_equal(original.coords['lon'], back.coords['lon']) + + def test_to_and_from_cdms2_ugrid(self): + """Unstructured grid""" + pytest.importorskip('cdms2') + + lon = DataArray(np.random.uniform(size=5), dims=['cell'], name='lon') + lat = DataArray(np.random.uniform(size=5), dims=['cell'], name='lat') + cell = DataArray(np.arange(5), dims=['cell'], name='cell') + original = DataArray(np.arange(5), dims=['cell'], + coords={'lon': lon, 'lat': lat, 'cell': cell}) + actual = original.to_cdms2() + self.assertItemsEqual(actual.getAxisIds(), original.dims) + assert_array_equal(original.coords['lon'], actual.getLongitude()) + assert_array_equal(original.coords['lat'], actual.getLatitude()) + + back = from_cdms2(actual) + self.assertItemsEqual(original.dims, back.dims) + self.assertItemsEqual(original.coords.keys(), back.coords.keys()) + assert_array_equal(original.coords['lat'], back.coords['lat']) + assert_array_equal(original.coords['lon'], back.coords['lon']) + def test_to_and_from_iris(self): try: import iris From 63cc96484270dee83e92391e49ad76b8ef3b40b3 Mon Sep 17 00:00:00 2001 From: Yohai Bar Sinai <6164157+yohai@users.noreply.github.com> Date: Wed, 4 Jul 2018 13:06:54 -0400 Subject: [PATCH 159/282] BUG: unnamed args in faceted line plots (#2259) * BUG: unnamed args in faceted line plots * minor change in commented code --- xarray/plot/plot.py | 3 ++- xarray/tests/test_plot.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 19b6420f35b..2a7fb08efda 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -289,7 +289,6 @@ def line(darray, *args, **kwargs): if row or col: allargs = locals().copy() allargs.update(allargs.pop('kwargs')) - allargs.update(allargs.pop('args')) return _line_facetgrid(**allargs) ndims = len(darray.dims) @@ -310,6 +309,8 @@ def line(darray, *args, **kwargs): yincrease = kwargs.pop('yincrease', True) add_legend = kwargs.pop('add_legend', True) _labels = kwargs.pop('_labels', True) + if args is (): + args = kwargs.pop('args', ()) ax = get_axis(figsize, size, aspect, ax) xplt, yplt, hueplt, xlabel, ylabel, huelabel = \ diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 4f8a3277f9d..986a2a93380 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1564,6 +1564,14 @@ def test_facetgrid_shape(self): g = self.darray.plot(row='col', col='row', hue='hue') assert g.axes.shape == (len(self.darray.col), len(self.darray.row)) + def test_unnamed_args(self): + g = self.darray.plot.line('o--', row='row', col='col', hue='hue') + lines = [q for q in g.axes.flat[0].get_children() + if isinstance(q, mpl.lines.Line2D)] + # passing 'o--' as argument should set marker and linestyle + assert lines[0].get_marker() == 'o' + assert lines[0].get_linestyle() == '--' + def test_default_labels(self): g = self.darray.plot(row='row', col='col', hue='hue') # Rightmost column should be labeled From 448c3f1ae919f94a2b594eaeb91c6fd950eca43f Mon Sep 17 00:00:00 2001 From: Ed Doddridge Date: Thu, 5 Jul 2018 15:01:16 -0400 Subject: [PATCH 160/282] Fix resample docstring seasonal average example (#2268) Closes #2232 --- xarray/core/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index d69c60eed56..3f934fcc769 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -630,12 +630,12 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, array([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.]) Coordinates: * time (time) datetime64[ns] 1999-12-15 2000-01-15 2000-02-15 ... - >>> da.resample(time="Q-DEC").mean() + >>> da.resample(time="QS-DEC").mean() array([ 1., 4., 7., 10.]) Coordinates: - * time (time) datetime64[ns] 2000-02-29 2000-05-31 2000-08-31 2000-11-30 - + * time (time) datetime64[ns] 1999-12-01 2000-03-01 2000-06-01 2000-09-01 + Upsample monthly time-series data to daily data: >>> da.resample(time='1D').interpolate('linear') From 1688a59803786a9d88eeb43aa4c935f7052d6a80 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 7 Jul 2018 17:55:30 -0700 Subject: [PATCH 161/282] update minimum versions and associated code cleanup (#2204) * update minimum versions and associated code cleanup * fix a few typos * pandas to 0.20 * dask min version to 0.16 * more min versions for ci * docs for pandas version * pandas ver 0.19 * revert back to using pytest.mark.skipif * bump min version of matplotlib to 1.5 (remove viridis colormap packaged in xarray) * pin distributed version as well * no distributed in compat build * skip distributed test for old versions of dask/distributed * skip a plotting test for older versions of mpl * whatsnew --- .travis.yml | 4 +- ci/requirements-py27-min.yml | 4 +- ci/requirements-py34.yml | 10 -- ci/requirements-py35.yml | 5 +- doc/installing.rst | 9 +- doc/whats-new.rst | 12 ++ setup.py | 2 +- xarray/core/duck_array_ops.py | 4 +- xarray/core/missing.py | 5 +- xarray/core/npcompat.py | 254 --------------------------- xarray/core/nputils.py | 6 +- xarray/plot/default_colormap.csv | 256 ---------------------------- xarray/plot/utils.py | 18 -- xarray/tests/__init__.py | 21 ++- xarray/tests/test_computation.py | 6 - xarray/tests/test_dask.py | 31 +--- xarray/tests/test_dataarray.py | 10 +- xarray/tests/test_dataset.py | 4 - xarray/tests/test_distributed.py | 4 +- xarray/tests/test_duck_array_ops.py | 8 +- xarray/tests/test_missing.py | 26 +-- xarray/tests/test_plot.py | 29 ++-- xarray/tests/test_ufuncs.py | 32 ++-- xarray/tests/test_variable.py | 13 +- 24 files changed, 79 insertions(+), 694 deletions(-) delete mode 100644 ci/requirements-py34.yml delete mode 100644 xarray/plot/default_colormap.csv diff --git a/.travis.yml b/.travis.yml index bd53edb0029..6df70e92954 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,6 @@ matrix: env: CONDA_ENV=py27-min - python: 2.7 env: CONDA_ENV=py27-cdat+iris+pynio - - python: 3.4 - env: CONDA_ENV=py34 - python: 3.5 env: CONDA_ENV=py35 - python: 3.6 @@ -102,7 +100,7 @@ install: script: # TODO: restore this check once the upstream pandas issue is fixed: - # https://github.com/pandas-dev/pandas/issues/21071 + # https://github.com/pandas-dev/pandas/issues/21071 # - python -OO -c "import xarray" - if [[ "$CONDA_ENV" == "docs" ]]; then conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc; diff --git a/ci/requirements-py27-min.yml b/ci/requirements-py27-min.yml index 50f6724ec51..118b629271e 100644 --- a/ci/requirements-py27-min.yml +++ b/ci/requirements-py27-min.yml @@ -4,8 +4,8 @@ dependencies: - pytest - flake8 - mock - - numpy==1.11 - - pandas==0.18.0 + - numpy=1.12 + - pandas=0.19 - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py34.yml b/ci/requirements-py34.yml deleted file mode 100644 index ba79e00bb12..00000000000 --- a/ci/requirements-py34.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: test_env -dependencies: - - python=3.4 - - bottleneck - - flake8 - - pandas - - pip: - - coveralls - - pytest-cov - - pytest diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index d3500bc5d10..9615aeba9aa 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -4,11 +4,10 @@ channels: dependencies: - python=3.5 - cftime - - dask - - distributed + - dask=0.16 - h5py - h5netcdf - - matplotlib + - matplotlib=1.5 - netcdf4 - pytest - flake8 diff --git a/doc/installing.rst b/doc/installing.rst index 31fc109ee2e..b3154c3d8bb 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -6,9 +6,9 @@ Installation Required dependencies --------------------- -- Python 2.7 [1]_, 3.4, 3.5, or 3.6 -- `numpy `__ (1.11 or later) -- `pandas `__ (0.18.0 or later) +- Python 2.7 [1]_, 3.5, or 3.6 +- `numpy `__ (1.12 or later) +- `pandas `__ (0.19.2 or later) Optional dependencies --------------------- @@ -45,13 +45,14 @@ For accelerating xarray For parallel computing ~~~~~~~~~~~~~~~~~~~~~~ -- `dask.array `__ (0.9.0 or later): required for +- `dask.array `__ (0.16 or later): required for :ref:`dask`. For plotting ~~~~~~~~~~~~ - `matplotlib `__: required for :ref:`plotting` + (1.5 or later) - `cartopy `__: recommended for :ref:`plot-maps` - `seaborn `__: for better diff --git a/doc/whats-new.rst b/doc/whats-new.rst index bd23386a460..af90ea7f9d3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -66,6 +66,18 @@ Bug fixes weren't monotonic (:issue:`2250`). By `Fabien Maussion `_. +Breaking changes +~~~~~~~~~~~~~~~~ + +- Xarray no longer supports python 3.4. Additionally, the minimum supported + versions of the following dependencies has been updated and/or clarified: + + - Pandas: 0.18 -> 0.19 + - NumPy: 1.11 -> 1.12 + - Dask: 0.9 -> 0.16 + - Matplotlib: unspecified -> 1.5 + + (:issue:`2204`). By `Joe Hamman `_. .. _whats-new.0.10.7: diff --git a/setup.py b/setup.py index 77c6083f52c..e35611e01b1 100644 --- a/setup.py +++ b/setup.py @@ -71,4 +71,4 @@ tests_require=TESTS_REQUIRE, url=URL, packages=find_packages(), - package_data={'xarray': ['tests/data/*', 'plot/default_colormap.csv']}) + package_data={'xarray': ['tests/data/*']}) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 065ac165a0d..3bd105064da 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -363,9 +363,9 @@ def f(values, axis=None, skipna=None, **kwargs): median = _create_nan_agg_method('median', numeric_only=True) prod = _create_nan_agg_method('prod', numeric_only=True, no_bottleneck=True) cumprod_1d = _create_nan_agg_method( - 'cumprod', numeric_only=True, np_compat=True, no_bottleneck=True) + 'cumprod', numeric_only=True, no_bottleneck=True) cumsum_1d = _create_nan_agg_method( - 'cumsum', numeric_only=True, np_compat=True, no_bottleneck=True) + 'cumsum', numeric_only=True, no_bottleneck=True) def _nd_cum_func(cum_func, array, axis, **kwargs): diff --git a/xarray/core/missing.py b/xarray/core/missing.py index 743627bb381..bec9e2e1931 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -8,7 +8,6 @@ from . import rolling from .computation import apply_ufunc -from .npcompat import flip from .pycompat import iteritems from .utils import is_scalar, OrderedSet from .variable import Variable, broadcast_variables @@ -245,13 +244,13 @@ def _bfill(arr, n=None, axis=-1): '''inverse of ffill''' import bottleneck as bn - arr = flip(arr, axis=axis) + arr = np.flip(arr, axis=axis) # fill arr = bn.push(arr, axis=axis, n=n) # reverse back to original - return flip(arr, axis=axis) + return np.flip(arr, axis=axis) def ffill(arr, dim=None, limit=None): diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index ec8adfffbf8..6d4db063b98 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -1,261 +1,7 @@ from __future__ import absolute_import, division, print_function -from distutils.version import LooseVersion - import numpy as np -if LooseVersion(np.__version__) >= LooseVersion('1.12'): - as_strided = np.lib.stride_tricks.as_strided -else: - def as_strided(x, shape=None, strides=None, subok=False, writeable=True): - array = np.lib.stride_tricks.as_strided(x, shape, strides, subok) - array.setflags(write=writeable) - return array - - -try: - from numpy import nancumsum, nancumprod, flip -except ImportError: # pragma: no cover - # Code copied from newer versions of NumPy (v1.12). - # Used under the terms of NumPy's license, see licenses/NUMPY_LICENSE. - - def _replace_nan(a, val): - """ - If `a` is of inexact type, make a copy of `a`, replace NaNs with - the `val` value, and return the copy together with a boolean mask - marking the locations where NaNs were present. If `a` is not of - inexact type, do nothing and return `a` together with a mask of None. - - Note that scalars will end up as array scalars, which is important - for using the result as the value of the out argument in some - operations. - - Parameters - ---------- - a : array-like - Input array. - val : float - NaN values are set to val before doing the operation. - - Returns - ------- - y : ndarray - If `a` is of inexact type, return a copy of `a` with the NaNs - replaced by the fill value, otherwise return `a`. - mask: {bool, None} - If `a` is of inexact type, return a boolean mask marking locations - of NaNs, otherwise return None. - - """ - is_new = not isinstance(a, np.ndarray) - if is_new: - a = np.array(a) - if not issubclass(a.dtype.type, np.inexact): - return a, None - if not is_new: - # need copy - a = np.array(a, subok=True) - - mask = np.isnan(a) - np.copyto(a, val, where=mask) - return a, mask - - def nancumsum(a, axis=None, dtype=None, out=None): - """ - Return the cumulative sum of array elements over a given axis treating - Not a Numbers (NaNs) as zero. The cumulative sum does not change when - NaNs are encountered and leading NaNs are replaced by zeros. - - Zeros are returned for slices that are all-NaN or empty. - - .. versionadded:: 1.12.0 - - Parameters - ---------- - a : array_like - Input array. - axis : int, optional - Axis along which the cumulative sum is computed. The default - (None) is to compute the cumsum over the flattened array. - dtype : dtype, optional - Type of the returned array and of the accumulator in which the - elements are summed. If `dtype` is not specified, it defaults - to the dtype of `a`, unless `a` has an integer dtype with a - precision less than that of the default platform integer. In - that case, the default platform integer is used. - out : ndarray, optional - Alternative output array in which to place the result. It must - have the same shape and buffer length as the expected output - but the type will be cast if necessary. See `doc.ufuncs` - (Section "Output arguments") for more details. - - Returns - ------- - nancumsum : ndarray. - A new array holding the result is returned unless `out` is - specified, in which it is returned. The result has the same - size as `a`, and the same shape as `a` if `axis` is not None - or `a` is a 1-d array. - - See Also - -------- - numpy.cumsum : Cumulative sum across array propagating NaNs. - isnan : Show which elements are NaN. - - Examples - -------- - >>> np.nancumsum(1) - array([1]) - >>> np.nancumsum([1]) - array([1]) - >>> np.nancumsum([1, np.nan]) - array([ 1., 1.]) - >>> a = np.array([[1, 2], [3, np.nan]]) - >>> np.nancumsum(a) - array([ 1., 3., 6., 6.]) - >>> np.nancumsum(a, axis=0) - array([[ 1., 2.], - [ 4., 2.]]) - >>> np.nancumsum(a, axis=1) - array([[ 1., 3.], - [ 3., 3.]]) - - """ - a, mask = _replace_nan(a, 0) - return np.cumsum(a, axis=axis, dtype=dtype, out=out) - - def nancumprod(a, axis=None, dtype=None, out=None): - """ - Return the cumulative product of array elements over a given axis - treating Not a Numbers (NaNs) as one. The cumulative product does not - change when NaNs are encountered and leading NaNs are replaced by ones. - - Ones are returned for slices that are all-NaN or empty. - - .. versionadded:: 1.12.0 - - Parameters - ---------- - a : array_like - Input array. - axis : int, optional - Axis along which the cumulative product is computed. By default - the input is flattened. - dtype : dtype, optional - Type of the returned array, as well as of the accumulator in which - the elements are multiplied. If *dtype* is not specified, it - defaults to the dtype of `a`, unless `a` has an integer dtype with - a precision less than that of the default platform integer. In - that case, the default platform integer is used instead. - out : ndarray, optional - Alternative output array in which to place the result. It must - have the same shape and buffer length as the expected output - but the type of the resulting values will be cast if necessary. - - Returns - ------- - nancumprod : ndarray - A new array holding the result is returned unless `out` is - specified, in which case it is returned. - - See Also - -------- - numpy.cumprod : Cumulative product across array propagating NaNs. - isnan : Show which elements are NaN. - - Examples - -------- - >>> np.nancumprod(1) - array([1]) - >>> np.nancumprod([1]) - array([1]) - >>> np.nancumprod([1, np.nan]) - array([ 1., 1.]) - >>> a = np.array([[1, 2], [3, np.nan]]) - >>> np.nancumprod(a) - array([ 1., 2., 6., 6.]) - >>> np.nancumprod(a, axis=0) - array([[ 1., 2.], - [ 3., 2.]]) - >>> np.nancumprod(a, axis=1) - array([[ 1., 2.], - [ 3., 3.]]) - - """ - a, mask = _replace_nan(a, 1) - return np.cumprod(a, axis=axis, dtype=dtype, out=out) - - def flip(m, axis): - """ - Reverse the order of elements in an array along the given axis. - - The shape of the array is preserved, but the elements are reordered. - - .. versionadded:: 1.12.0 - - Parameters - ---------- - m : array_like - Input array. - axis : integer - Axis in array, which entries are reversed. - - - Returns - ------- - out : array_like - A view of `m` with the entries of axis reversed. Since a view is - returned, this operation is done in constant time. - - See Also - -------- - flipud : Flip an array vertically (axis=0). - fliplr : Flip an array horizontally (axis=1). - - Notes - ----- - flip(m, 0) is equivalent to flipud(m). - flip(m, 1) is equivalent to fliplr(m). - flip(m, n) corresponds to ``m[...,::-1,...]`` with ``::-1`` at index n. - - Examples - -------- - >>> A = np.arange(8).reshape((2,2,2)) - >>> A - array([[[0, 1], - [2, 3]], - - [[4, 5], - [6, 7]]]) - - >>> flip(A, 0) - array([[[4, 5], - [6, 7]], - - [[0, 1], - [2, 3]]]) - - >>> flip(A, 1) - array([[[2, 3], - [0, 1]], - - [[6, 7], - [4, 5]]]) - - >>> A = np.random.randn(3,4,5) - >>> np.all(flip(A,2) == A[:,:,::-1,...]) - True - """ - if not hasattr(m, 'ndim'): - m = np.asarray(m) - indexer = [slice(None)] * m.ndim - try: - indexer[axis] = slice(None, None, -1) - except IndexError: - raise ValueError("axis=%i is invalid for the %i-dimensional " - "input array" % (axis, m.ndim)) - return m[tuple(indexer)] - try: from numpy import isin except ImportError: diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 4ca1f9390eb..6df2d34bfe3 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -5,8 +5,6 @@ import numpy as np import pandas as pd -from . import npcompat - def _validate_axis(data, axis): ndim = data.ndim @@ -194,6 +192,6 @@ def _rolling_window(a, window, axis=-1): shape = a.shape[:-1] + (a.shape[-1] - window + 1, window) strides = a.strides + (a.strides[-1],) - rolling = npcompat.as_strided(a, shape=shape, strides=strides, - writeable=False) + rolling = np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides, + writeable=False) return np.swapaxes(rolling, -2, axis) diff --git a/xarray/plot/default_colormap.csv b/xarray/plot/default_colormap.csv deleted file mode 100644 index de9632e3f26..00000000000 --- a/xarray/plot/default_colormap.csv +++ /dev/null @@ -1,256 +0,0 @@ -0.26700401,0.00487433,0.32941519 -0.26851048,0.00960483,0.33542652 -0.26994384,0.01462494,0.34137895 -0.27130489,0.01994186,0.34726862 -0.27259384,0.02556309,0.35309303 -0.27380934,0.03149748,0.35885256 -0.27495242,0.03775181,0.36454323 -0.27602238,0.04416723,0.37016418 -0.2770184,0.05034437,0.37571452 -0.27794143,0.05632444,0.38119074 -0.27879067,0.06214536,0.38659204 -0.2795655,0.06783587,0.39191723 -0.28026658,0.07341724,0.39716349 -0.28089358,0.07890703,0.40232944 -0.28144581,0.0843197,0.40741404 -0.28192358,0.08966622,0.41241521 -0.28232739,0.09495545,0.41733086 -0.28265633,0.10019576,0.42216032 -0.28291049,0.10539345,0.42690202 -0.28309095,0.11055307,0.43155375 -0.28319704,0.11567966,0.43611482 -0.28322882,0.12077701,0.44058404 -0.28318684,0.12584799,0.44496 -0.283072,0.13089477,0.44924127 -0.28288389,0.13592005,0.45342734 -0.28262297,0.14092556,0.45751726 -0.28229037,0.14591233,0.46150995 -0.28188676,0.15088147,0.46540474 -0.28141228,0.15583425,0.46920128 -0.28086773,0.16077132,0.47289909 -0.28025468,0.16569272,0.47649762 -0.27957399,0.17059884,0.47999675 -0.27882618,0.1754902,0.48339654 -0.27801236,0.18036684,0.48669702 -0.27713437,0.18522836,0.48989831 -0.27619376,0.19007447,0.49300074 -0.27519116,0.1949054,0.49600488 -0.27412802,0.19972086,0.49891131 -0.27300596,0.20452049,0.50172076 -0.27182812,0.20930306,0.50443413 -0.27059473,0.21406899,0.50705243 -0.26930756,0.21881782,0.50957678 -0.26796846,0.22354911,0.5120084 -0.26657984,0.2282621,0.5143487 -0.2651445,0.23295593,0.5165993 -0.2636632,0.23763078,0.51876163 -0.26213801,0.24228619,0.52083736 -0.26057103,0.2469217,0.52282822 -0.25896451,0.25153685,0.52473609 -0.25732244,0.2561304,0.52656332 -0.25564519,0.26070284,0.52831152 -0.25393498,0.26525384,0.52998273 -0.25219404,0.26978306,0.53157905 -0.25042462,0.27429024,0.53310261 -0.24862899,0.27877509,0.53455561 -0.2468114,0.28323662,0.53594093 -0.24497208,0.28767547,0.53726018 -0.24311324,0.29209154,0.53851561 -0.24123708,0.29648471,0.53970946 -0.23934575,0.30085494,0.54084398 -0.23744138,0.30520222,0.5419214 -0.23552606,0.30952657,0.54294396 -0.23360277,0.31382773,0.54391424 -0.2316735,0.3181058,0.54483444 -0.22973926,0.32236127,0.54570633 -0.22780192,0.32659432,0.546532 -0.2258633,0.33080515,0.54731353 -0.22392515,0.334994,0.54805291 -0.22198915,0.33916114,0.54875211 -0.22005691,0.34330688,0.54941304 -0.21812995,0.34743154,0.55003755 -0.21620971,0.35153548,0.55062743 -0.21429757,0.35561907,0.5511844 -0.21239477,0.35968273,0.55171011 -0.2105031,0.36372671,0.55220646 -0.20862342,0.36775151,0.55267486 -0.20675628,0.37175775,0.55311653 -0.20490257,0.37574589,0.55353282 -0.20306309,0.37971644,0.55392505 -0.20123854,0.38366989,0.55429441 -0.1994295,0.38760678,0.55464205 -0.1976365,0.39152762,0.55496905 -0.19585993,0.39543297,0.55527637 -0.19410009,0.39932336,0.55556494 -0.19235719,0.40319934,0.55583559 -0.19063135,0.40706148,0.55608907 -0.18892259,0.41091033,0.55632606 -0.18723083,0.41474645,0.55654717 -0.18555593,0.4185704,0.55675292 -0.18389763,0.42238275,0.55694377 -0.18225561,0.42618405,0.5571201 -0.18062949,0.42997486,0.55728221 -0.17901879,0.43375572,0.55743035 -0.17742298,0.4375272,0.55756466 -0.17584148,0.44128981,0.55768526 -0.17427363,0.4450441,0.55779216 -0.17271876,0.4487906,0.55788532 -0.17117615,0.4525298,0.55796464 -0.16964573,0.45626209,0.55803034 -0.16812641,0.45998802,0.55808199 -0.1666171,0.46370813,0.55811913 -0.16511703,0.4674229,0.55814141 -0.16362543,0.47113278,0.55814842 -0.16214155,0.47483821,0.55813967 -0.16066467,0.47853961,0.55811466 -0.15919413,0.4822374,0.5580728 -0.15772933,0.48593197,0.55801347 -0.15626973,0.4896237,0.557936 -0.15481488,0.49331293,0.55783967 -0.15336445,0.49700003,0.55772371 -0.1519182,0.50068529,0.55758733 -0.15047605,0.50436904,0.55742968 -0.14903918,0.50805136,0.5572505 -0.14760731,0.51173263,0.55704861 -0.14618026,0.51541316,0.55682271 -0.14475863,0.51909319,0.55657181 -0.14334327,0.52277292,0.55629491 -0.14193527,0.52645254,0.55599097 -0.14053599,0.53013219,0.55565893 -0.13914708,0.53381201,0.55529773 -0.13777048,0.53749213,0.55490625 -0.1364085,0.54117264,0.55448339 -0.13506561,0.54485335,0.55402906 -0.13374299,0.54853458,0.55354108 -0.13244401,0.55221637,0.55301828 -0.13117249,0.55589872,0.55245948 -0.1299327,0.55958162,0.55186354 -0.12872938,0.56326503,0.55122927 -0.12756771,0.56694891,0.55055551 -0.12645338,0.57063316,0.5498411 -0.12539383,0.57431754,0.54908564 -0.12439474,0.57800205,0.5482874 -0.12346281,0.58168661,0.54744498 -0.12260562,0.58537105,0.54655722 -0.12183122,0.58905521,0.54562298 -0.12114807,0.59273889,0.54464114 -0.12056501,0.59642187,0.54361058 -0.12009154,0.60010387,0.54253043 -0.11973756,0.60378459,0.54139999 -0.11951163,0.60746388,0.54021751 -0.11942341,0.61114146,0.53898192 -0.11948255,0.61481702,0.53769219 -0.11969858,0.61849025,0.53634733 -0.12008079,0.62216081,0.53494633 -0.12063824,0.62582833,0.53348834 -0.12137972,0.62949242,0.53197275 -0.12231244,0.63315277,0.53039808 -0.12344358,0.63680899,0.52876343 -0.12477953,0.64046069,0.52706792 -0.12632581,0.64410744,0.52531069 -0.12808703,0.64774881,0.52349092 -0.13006688,0.65138436,0.52160791 -0.13226797,0.65501363,0.51966086 -0.13469183,0.65863619,0.5176488 -0.13733921,0.66225157,0.51557101 -0.14020991,0.66585927,0.5134268 -0.14330291,0.66945881,0.51121549 -0.1466164,0.67304968,0.50893644 -0.15014782,0.67663139,0.5065889 -0.15389405,0.68020343,0.50417217 -0.15785146,0.68376525,0.50168574 -0.16201598,0.68731632,0.49912906 -0.1663832,0.69085611,0.49650163 -0.1709484,0.69438405,0.49380294 -0.17570671,0.6978996,0.49103252 -0.18065314,0.70140222,0.48818938 -0.18578266,0.70489133,0.48527326 -0.19109018,0.70836635,0.48228395 -0.19657063,0.71182668,0.47922108 -0.20221902,0.71527175,0.47608431 -0.20803045,0.71870095,0.4728733 -0.21400015,0.72211371,0.46958774 -0.22012381,0.72550945,0.46622638 -0.2263969,0.72888753,0.46278934 -0.23281498,0.73224735,0.45927675 -0.2393739,0.73558828,0.45568838 -0.24606968,0.73890972,0.45202405 -0.25289851,0.74221104,0.44828355 -0.25985676,0.74549162,0.44446673 -0.26694127,0.74875084,0.44057284 -0.27414922,0.75198807,0.4366009 -0.28147681,0.75520266,0.43255207 -0.28892102,0.75839399,0.42842626 -0.29647899,0.76156142,0.42422341 -0.30414796,0.76470433,0.41994346 -0.31192534,0.76782207,0.41558638 -0.3198086,0.77091403,0.41115215 -0.3277958,0.77397953,0.40664011 -0.33588539,0.7770179,0.40204917 -0.34407411,0.78002855,0.39738103 -0.35235985,0.78301086,0.39263579 -0.36074053,0.78596419,0.38781353 -0.3692142,0.78888793,0.38291438 -0.37777892,0.79178146,0.3779385 -0.38643282,0.79464415,0.37288606 -0.39517408,0.79747541,0.36775726 -0.40400101,0.80027461,0.36255223 -0.4129135,0.80304099,0.35726893 -0.42190813,0.80577412,0.35191009 -0.43098317,0.80847343,0.34647607 -0.44013691,0.81113836,0.3409673 -0.44936763,0.81376835,0.33538426 -0.45867362,0.81636288,0.32972749 -0.46805314,0.81892143,0.32399761 -0.47750446,0.82144351,0.31819529 -0.4870258,0.82392862,0.31232133 -0.49661536,0.82637633,0.30637661 -0.5062713,0.82878621,0.30036211 -0.51599182,0.83115784,0.29427888 -0.52577622,0.83349064,0.2881265 -0.5356211,0.83578452,0.28190832 -0.5455244,0.83803918,0.27562602 -0.55548397,0.84025437,0.26928147 -0.5654976,0.8424299,0.26287683 -0.57556297,0.84456561,0.25641457 -0.58567772,0.84666139,0.24989748 -0.59583934,0.84871722,0.24332878 -0.60604528,0.8507331,0.23671214 -0.61629283,0.85270912,0.23005179 -0.62657923,0.85464543,0.22335258 -0.63690157,0.85654226,0.21662012 -0.64725685,0.85839991,0.20986086 -0.65764197,0.86021878,0.20308229 -0.66805369,0.86199932,0.19629307 -0.67848868,0.86374211,0.18950326 -0.68894351,0.86544779,0.18272455 -0.69941463,0.86711711,0.17597055 -0.70989842,0.86875092,0.16925712 -0.72039115,0.87035015,0.16260273 -0.73088902,0.87191584,0.15602894 -0.74138803,0.87344918,0.14956101 -0.75188414,0.87495143,0.14322828 -0.76237342,0.87642392,0.13706449 -0.77285183,0.87786808,0.13110864 -0.78331535,0.87928545,0.12540538 -0.79375994,0.88067763,0.12000532 -0.80418159,0.88204632,0.11496505 -0.81457634,0.88339329,0.11034678 -0.82494028,0.88472036,0.10621724 -0.83526959,0.88602943,0.1026459 -0.84556056,0.88732243,0.09970219 -0.8558096,0.88860134,0.09745186 -0.86601325,0.88986815,0.09595277 -0.87616824,0.89112487,0.09525046 -0.88627146,0.89237353,0.09537439 -0.89632002,0.89361614,0.09633538 -0.90631121,0.89485467,0.09812496 -0.91624212,0.89609127,0.1007168 -0.92610579,0.89732977,0.10407067 -0.93590444,0.8985704,0.10813094 -0.94563626,0.899815,0.11283773 -0.95529972,0.90106534,0.11812832 -0.96489353,0.90232311,0.12394051 -0.97441665,0.90358991,0.13021494 -0.98386829,0.90486726,0.13689671 -0.99324789,0.90615657,0.1439362 diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 6846c553b8b..4b9645e02d5 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -13,20 +13,6 @@ ROBUST_PERCENTILE = 2.0 -def _load_default_cmap(fname='default_colormap.csv'): - """ - Returns viridis color map - """ - from matplotlib.colors import LinearSegmentedColormap - - # Not sure what the first arg here should be - f = pkg_resources.resource_stream(__name__, fname) - cm_data = pd.read_csv(f, header=None).values - f.close() - - return LinearSegmentedColormap.from_list('viridis', cm_data) - - def import_seaborn(): '''import seaborn and handle deprecation of apionly module''' with warnings.catch_warnings(record=True) as w: @@ -226,10 +212,6 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, else: cmap = "viridis" - # Allow viridis before matplotlib 1.5 - if cmap == "viridis": - cmap = _load_default_cmap() - # Handle discrete levels if levels is not None: if is_scalar(levels): diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index e93d9a80145..3b4d69a35f7 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -54,15 +54,13 @@ def _importorskip(modname, minversion=None): raise ImportError('Minimum version not satisfied') except ImportError: has = False - # TODO: use pytest.skipif instead of unittest.skipUnless - # Using `unittest.skipUnless` is a temporary workaround for pytest#568, - # wherein class decorators stain inherited classes. - # xref: xarray#1531, implemented in xarray #1557. - func = unittest.skipUnless(has, reason='requires {}'.format(modname)) + func = pytest.mark.skipif(not has, reason='requires {}'.format(modname)) return has, func has_matplotlib, requires_matplotlib = _importorskip('matplotlib') +has_matplotlib2, requires_matplotlib2 = _importorskip('matplotlib', + minversion='2') has_scipy, requires_scipy = _importorskip('scipy') has_pydap, requires_pydap = _importorskip('pydap.client') has_netCDF4, requires_netCDF4 = _importorskip('netCDF4') @@ -75,15 +73,15 @@ def _importorskip(modname, minversion=None): has_rasterio, requires_rasterio = _importorskip('rasterio') has_pathlib, requires_pathlib = _importorskip('pathlib') has_zarr, requires_zarr = _importorskip('zarr', minversion='2.2') -has_np112, requires_np112 = _importorskip('numpy', minversion='1.12.0') +has_np113, requires_np113 = _importorskip('numpy', minversion='1.13.0') # some special cases has_scipy_or_netCDF4 = has_scipy or has_netCDF4 -requires_scipy_or_netCDF4 = unittest.skipUnless( - has_scipy_or_netCDF4, reason='requires scipy or netCDF4') +requires_scipy_or_netCDF4 = pytest.mark.skipif( + not has_scipy_or_netCDF4, reason='requires scipy or netCDF4') has_cftime_or_netCDF4 = has_cftime or has_netCDF4 -requires_cftime_or_netCDF4 = unittest.skipUnless( - has_cftime_or_netCDF4, reason='requires cftime or netCDF4') +requires_cftime_or_netCDF4 = pytest.mark.skipif( + not has_cftime_or_netCDF4, reason='requires cftime or netCDF4') if not has_pathlib: has_pathlib, requires_pathlib = _importorskip('pathlib2') if has_dask: @@ -97,7 +95,8 @@ def _importorskip(modname, minversion=None): has_seaborn = True except ImportError: has_seaborn = False -requires_seaborn = unittest.skipUnless(has_seaborn, reason='requires seaborn') +requires_seaborn = pytest.mark.skipif(not has_seaborn, + reason='requires seaborn') try: _SKIP_FLAKY = not pytest.config.getoption("--run-flaky") diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index a802b91a3db..e30e7e31390 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -726,9 +726,6 @@ def pandas_median(x): def test_vectorize(): - if LooseVersion(np.__version__) < LooseVersion('1.12.0'): - pytest.skip('numpy 1.12 or later to support vectorize=True.') - data_array = xr.DataArray([[0, 1, 2], [1, 2, 3]], dims=('x', 'y')) expected = xr.DataArray([1, 2], dims=['x']) actual = apply_ufunc(pandas_median, data_array, @@ -739,9 +736,6 @@ def test_vectorize(): @requires_dask def test_vectorize_dask(): - if LooseVersion(np.__version__) < LooseVersion('1.12.0'): - pytest.skip('numpy 1.12 or later to support vectorize=True.') - data_array = xr.DataArray([[0, 1, 2], [1, 2, 3]], dims=('x', 'y')) expected = xr.DataArray([1, 2], dims=['x']) actual = apply_ufunc(pandas_median, data_array.chunk({'x': 1}), diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index ee5b3514348..f6c47cce8d8 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1,7 +1,6 @@ from __future__ import absolute_import, division, print_function import pickle -from distutils.version import LooseVersion from textwrap import dedent import numpy as np @@ -208,8 +207,6 @@ def test_bivariate_ufunc(self): self.assertLazyAndAllClose(np.maximum(u, 0), xu.maximum(v, 0)) self.assertLazyAndAllClose(np.maximum(u, 0), xu.maximum(0, v)) - @pytest.mark.skipif(LooseVersion(dask.__version__) <= '0.15.4', - reason='Need dask 0.16 for new interface') def test_compute(self): u = self.eager_var v = self.lazy_var @@ -220,8 +217,6 @@ def test_compute(self): assert ((u + 1).data == v2.data).all() - @pytest.mark.skipif(LooseVersion(dask.__version__) <= '0.15.4', - reason='Need dask 0.16 for new interface') def test_persist(self): u = self.eager_var v = self.lazy_var + 1 @@ -281,8 +276,6 @@ def test_lazy_array(self): actual = xr.concat([v[:2], v[2:]], 'x') self.assertLazyAndAllClose(u, actual) - @pytest.mark.skipif(LooseVersion(dask.__version__) <= '0.15.4', - reason='Need dask 0.16 for new interface') def test_compute(self): u = self.eager_array v = self.lazy_array @@ -293,8 +286,6 @@ def test_compute(self): assert ((u + 1).data == v2.data).all() - @pytest.mark.skipif(LooseVersion(dask.__version__) <= '0.15.4', - reason='Need dask 0.16 for new interface') def test_persist(self): u = self.eager_array v = self.lazy_array + 1 @@ -384,10 +375,6 @@ def test_concat_loads_variables(self): assert ds3['c'].data is c3 def test_groupby(self): - if LooseVersion(dask.__version__) == LooseVersion('0.15.3'): - pytest.xfail('upstream bug in dask: ' - 'https://github.com/dask/dask/issues/2718') - u = self.eager_array v = self.lazy_array @@ -779,12 +766,8 @@ def build_dask_array(name): # test both the perist method and the dask.persist function # the dask.persist function requires a new version of dask -@pytest.mark.parametrize('persist', [ - lambda x: x.persist(), - pytest.mark.skipif(LooseVersion(dask.__version__) <= '0.15.4', - lambda x: dask.persist(x)[0], - reason='Need Dask 0.16') -]) +@pytest.mark.parametrize('persist', [lambda x: x.persist(), + lambda x: dask.persist(x)[0]]) def test_persist_Dataset(persist): ds = Dataset({'foo': ('x', range(5)), 'bar': ('x', range(5))}).chunk() @@ -797,12 +780,8 @@ def test_persist_Dataset(persist): assert len(ds.foo.data.dask) == n # doesn't mutate in place -@pytest.mark.parametrize('persist', [ - lambda x: x.persist(), - pytest.mark.skipif(LooseVersion(dask.__version__) <= '0.15.4', - lambda x: dask.persist(x)[0], - reason='Need Dask 0.16') -]) +@pytest.mark.parametrize('persist', [lambda x: x.persist(), + lambda x: dask.persist(x)[0]]) def test_persist_DataArray(persist): x = da.arange(10, chunks=(5,)) y = DataArray(x) @@ -815,8 +794,6 @@ def test_persist_DataArray(persist): assert len(zz.data.dask) == zz.data.npartitions -@pytest.mark.skipif(LooseVersion(dask.__version__) <= '0.15.4', - reason='Need dask 0.16 for new interface') def test_dataarray_with_dask_coords(): import toolz x = xr.Variable('x', da.arange(8, chunks=(4,))) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index df09a1e58df..e0b1496c7bf 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2,7 +2,6 @@ import pickle from copy import deepcopy -from distutils.version import LooseVersion from textwrap import dedent import warnings @@ -20,7 +19,7 @@ from xarray.tests import ( ReturnItem, TestCase, assert_allclose, assert_array_equal, assert_equal, assert_identical, raises_regex, requires_bottleneck, requires_cftime, - requires_dask, requires_scipy, source_ndarray, unittest) + requires_dask, requires_np113, requires_scipy, source_ndarray, unittest) class TestDataArray(TestCase): @@ -3379,9 +3378,6 @@ def test_sortby(self): actual = da.sortby([day, dax]) assert_equal(actual, expected) - if LooseVersion(np.__version__) < LooseVersion('1.11.0'): - pytest.skip('numpy 1.11.0 or later to support object data-type.') - expected = sorted1d actual = da.sortby('x') assert_equal(actual, expected) @@ -3647,9 +3643,7 @@ def test_rolling_reduce(da, center, min_periods, window, name): assert actual.dims == expected.dims -@pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion('1.13'), - reason='Old numpy does not support nansum / nanmax for ' - 'object typed arrays.') +@requires_np113 @pytest.mark.parametrize('center', (True, False)) @pytest.mark.parametrize('min_periods', (None, 1, 2, 3)) @pytest.mark.parametrize('window', (1, 2, 3, 4)) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index a0d316d74dc..4aa99b8ee5a 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2,7 +2,6 @@ from __future__ import absolute_import, division, print_function from copy import copy, deepcopy -from distutils.version import LooseVersion from io import StringIO from textwrap import dedent import warnings @@ -4036,9 +4035,6 @@ def test_sortby(self): actual = ds.sortby(ds['A']) assert "DataArray is not 1-D" in str(excinfo.value) - if LooseVersion(np.__version__) < LooseVersion('1.11.0'): - pytest.skip('numpy 1.11.0 or later to support object data-type.') - expected = sorted1d actual = ds.sortby('x') assert_equal(actual, expected) diff --git a/xarray/tests/test_distributed.py b/xarray/tests/test_distributed.py index 8679e892be4..32035afdc57 100644 --- a/xarray/tests/test_distributed.py +++ b/xarray/tests/test_distributed.py @@ -7,8 +7,8 @@ import pytest -dask = pytest.importorskip('dask') # isort:skip -distributed = pytest.importorskip('distributed') # isort:skip +dask = pytest.importorskip('dask', minversion='0.18') # isort:skip +distributed = pytest.importorskip('distributed', minversion='1.21') # isort:skip from dask import array from dask.distributed import Client, Lock diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 3f4adee6713..f3f93491822 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -1,7 +1,5 @@ from __future__ import absolute_import, division, print_function -from distutils.version import LooseVersion - import numpy as np import pytest from numpy import array, nan @@ -16,7 +14,8 @@ from xarray.testing import assert_allclose, assert_equal from . import ( - TestCase, assert_array_equal, has_dask, raises_regex, requires_dask) + TestCase, assert_array_equal, has_dask, has_np113, raises_regex, + requires_dask) class TestOps(TestCase): @@ -263,8 +262,7 @@ def test_reduce(dim_num, dtype, dask, func, skipna, aggdim): warnings.filterwarnings('ignore', 'All-NaN slice') warnings.filterwarnings('ignore', 'invalid value encountered in') - if (LooseVersion(np.__version__) >= LooseVersion('1.13.0') and - da.dtype.kind == 'O' and skipna): + if has_np113 and da.dtype.kind == 'O' and skipna: # Numpy < 1.13 does not handle object-type array. try: if skipna: diff --git a/xarray/tests/test_missing.py b/xarray/tests/test_missing.py index 1dde95adf42..5c7e384c789 100644 --- a/xarray/tests/test_missing.py +++ b/xarray/tests/test_missing.py @@ -12,7 +12,7 @@ from xarray.core.pycompat import dask_array_type from xarray.tests import ( assert_array_equal, assert_equal, raises_regex, requires_bottleneck, - requires_dask, requires_np112, requires_scipy) + requires_dask, requires_scipy) @pytest.fixture @@ -67,7 +67,6 @@ def make_interpolate_example_data(shape, frac_nan, seed=12345, return da, df -@requires_np112 @requires_scipy def test_interpolate_pd_compat(): shapes = [(8, 8), (1, 20), (20, 1), (100, 100)] @@ -93,7 +92,6 @@ def test_interpolate_pd_compat(): np.testing.assert_allclose(actual.values, expected.values) -@requires_np112 @requires_scipy def test_scipy_methods_function(): for method in ['barycentric', 'krog', 'pchip', 'spline', 'akima']: @@ -105,7 +103,6 @@ def test_scipy_methods_function(): assert (da.count('time') <= actual.count('time')).all() -@requires_np112 @requires_scipy def test_interpolate_pd_compat_non_uniform_index(): shapes = [(8, 8), (1, 20), (20, 1), (100, 100)] @@ -134,7 +131,6 @@ def test_interpolate_pd_compat_non_uniform_index(): np.testing.assert_allclose(actual.values, expected.values) -@requires_np112 @requires_scipy def test_interpolate_pd_compat_polynomial(): shapes = [(8, 8), (1, 20), (20, 1), (100, 100)] @@ -154,7 +150,6 @@ def test_interpolate_pd_compat_polynomial(): np.testing.assert_allclose(actual.values, expected.values) -@requires_np112 @requires_scipy def test_interpolate_unsorted_index_raises(): vals = np.array([1, 2, 3], dtype=np.float64) @@ -195,7 +190,6 @@ def test_interpolate_2d_coord_raises(): da.interpolate_na(dim='a', use_coordinate='x') -@requires_np112 @requires_scipy def test_interpolate_kwargs(): da = xr.DataArray(np.array([4, 5, np.nan], dtype=np.float64), dims='x') @@ -208,7 +202,6 @@ def test_interpolate_kwargs(): assert_equal(actual, expected) -@requires_np112 def test_interpolate(): vals = np.array([1, 2, 3, 4, 5, 6], dtype=np.float64) @@ -222,7 +215,6 @@ def test_interpolate(): assert_equal(actual, expected) -@requires_np112 def test_interpolate_nonans(): vals = np.array([1, 2, 3, 4, 5, 6], dtype=np.float64) @@ -231,7 +223,6 @@ def test_interpolate_nonans(): assert_equal(actual, expected) -@requires_np112 @requires_scipy def test_interpolate_allnans(): vals = np.full(6, np.nan, dtype=np.float64) @@ -241,7 +232,6 @@ def test_interpolate_allnans(): assert_equal(actual, expected) -@requires_np112 @requires_bottleneck def test_interpolate_limits(): da = xr.DataArray(np.array([1, 2, np.nan, np.nan, np.nan, 6], @@ -257,7 +247,6 @@ def test_interpolate_limits(): assert_equal(actual, expected) -@requires_np112 @requires_scipy def test_interpolate_methods(): for method in ['linear', 'nearest', 'zero', 'slinear', 'quadratic', @@ -273,7 +262,6 @@ def test_interpolate_methods(): @requires_scipy -@requires_np112 def test_interpolators(): for method, interpolator in [('linear', NumpyInterpolator), ('linear', ScipyInterpolator), @@ -287,7 +275,6 @@ def test_interpolators(): assert pd.isnull(out).sum() == 0 -@requires_np112 def test_interpolate_use_coordinate(): xc = xr.Variable('x', [100, 200, 300, 400, 500, 600]) da = xr.DataArray(np.array([1, 2, np.nan, np.nan, np.nan, 6], @@ -310,7 +297,6 @@ def test_interpolate_use_coordinate(): assert_equal(actual, expected) -@requires_np112 @requires_dask def test_interpolate_dask(): da, _ = make_interpolate_example_data((40, 40), 0.5) @@ -328,7 +314,6 @@ def test_interpolate_dask(): assert_equal(actual, expected) -@requires_np112 @requires_dask def test_interpolate_dask_raises_for_invalid_chunk_dim(): da, _ = make_interpolate_example_data((40, 40), 0.5) @@ -337,7 +322,6 @@ def test_interpolate_dask_raises_for_invalid_chunk_dim(): da.interpolate_na('time') -@requires_np112 @requires_bottleneck def test_ffill(): da = xr.DataArray(np.array([4, 5, np.nan], dtype=np.float64), dims='x') @@ -346,7 +330,6 @@ def test_ffill(): assert_equal(actual, expected) -@requires_np112 @requires_bottleneck @requires_dask def test_ffill_dask(): @@ -384,7 +367,6 @@ def test_bfill_dask(): @requires_bottleneck -@requires_np112 def test_ffill_bfill_nonans(): vals = np.array([1, 2, 3, 4, 5, 6], dtype=np.float64) @@ -398,7 +380,6 @@ def test_ffill_bfill_nonans(): @requires_bottleneck -@requires_np112 def test_ffill_bfill_allnans(): vals = np.full(6, np.nan, dtype=np.float64) @@ -412,14 +393,12 @@ def test_ffill_bfill_allnans(): @requires_bottleneck -@requires_np112 def test_ffill_functions(da): result = da.ffill('time') assert result.isnull().sum() == 0 @requires_bottleneck -@requires_np112 def test_ffill_limit(): da = xr.DataArray( [0, np.nan, np.nan, np.nan, np.nan, 3, 4, 5, np.nan, 6, 7], @@ -433,7 +412,6 @@ def test_ffill_limit(): [0, 0, np.nan, np.nan, np.nan, 3, 4, 5, 5, 6, 7], dims='time') -@requires_np112 def test_interpolate_dataset(ds): actual = ds.interpolate_na(dim='time') # no missing values in var1 @@ -444,12 +422,10 @@ def test_interpolate_dataset(ds): @requires_bottleneck -@requires_np112 def test_ffill_dataset(ds): ds.ffill(dim='time') @requires_bottleneck -@requires_np112 def test_bfill_dataset(ds): ds.ffill(dim='time') diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 986a2a93380..90d30946c9c 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -17,7 +17,8 @@ from . import ( TestCase, assert_array_equal, assert_equal, raises_regex, - requires_matplotlib, requires_seaborn, requires_cftime) + requires_matplotlib, requires_matplotlib2, requires_seaborn, + requires_cftime) # import mpl and change the backend before other mpl imports try: @@ -283,6 +284,7 @@ def test_convenient_facetgrid(self): d[0].plot(x='x', y='y', col='z', ax=plt.gca()) @pytest.mark.slow + @requires_matplotlib2 def test_subplot_kws(self): a = easy_array((10, 15, 4)) d = DataArray(a, dims=['y', 'x', 'z']) @@ -295,12 +297,9 @@ def test_subplot_kws(self): cmap='cool', subplot_kws=dict(facecolor='r')) for ax in g.axes.flat: - try: - # mpl V2 - assert ax.get_facecolor()[0:3] == \ - mpl.colors.to_rgb('r') - except AttributeError: - assert ax.get_axis_bgcolor() == 'r' + # mpl V2 + assert ax.get_facecolor()[0:3] == \ + mpl.colors.to_rgb('r') @pytest.mark.slow def test_plot_size(self): @@ -462,7 +461,7 @@ def test_robust(self): cmap_params = _determine_cmap_params(self.data, robust=True) assert cmap_params['vmin'] == np.percentile(self.data, 2) assert cmap_params['vmax'] == np.percentile(self.data, 98) - assert cmap_params['cmap'].name == 'viridis' + assert cmap_params['cmap'] == 'viridis' assert cmap_params['extend'] == 'both' assert cmap_params['levels'] is None assert cmap_params['norm'] is None @@ -546,7 +545,7 @@ def test_divergentcontrol(self): cmap_params = _determine_cmap_params(pos) assert cmap_params['vmin'] == 0 assert cmap_params['vmax'] == 1 - assert cmap_params['cmap'].name == "viridis" + assert cmap_params['cmap'] == "viridis" # Default with negative data will be a divergent cmap cmap_params = _determine_cmap_params(neg) @@ -558,17 +557,17 @@ def test_divergentcontrol(self): cmap_params = _determine_cmap_params(neg, vmin=-0.1, center=False) assert cmap_params['vmin'] == -0.1 assert cmap_params['vmax'] == 0.9 - assert cmap_params['cmap'].name == "viridis" + assert cmap_params['cmap'] == "viridis" cmap_params = _determine_cmap_params(neg, vmax=0.5, center=False) assert cmap_params['vmin'] == -0.1 assert cmap_params['vmax'] == 0.5 - assert cmap_params['cmap'].name == "viridis" + assert cmap_params['cmap'] == "viridis" # Setting center=False too cmap_params = _determine_cmap_params(neg, center=False) assert cmap_params['vmin'] == -0.1 assert cmap_params['vmax'] == 0.9 - assert cmap_params['cmap'].name == "viridis" + assert cmap_params['cmap'] == "viridis" # However, I should still be able to set center and have a div cmap cmap_params = _determine_cmap_params(neg, center=0) @@ -598,17 +597,17 @@ def test_divergentcontrol(self): cmap_params = _determine_cmap_params(pos, vmin=0.1) assert cmap_params['vmin'] == 0.1 assert cmap_params['vmax'] == 1 - assert cmap_params['cmap'].name == "viridis" + assert cmap_params['cmap'] == "viridis" cmap_params = _determine_cmap_params(pos, vmax=0.5) assert cmap_params['vmin'] == 0 assert cmap_params['vmax'] == 0.5 - assert cmap_params['cmap'].name == "viridis" + assert cmap_params['cmap'] == "viridis" # If both vmin and vmax are provided, output is non-divergent cmap_params = _determine_cmap_params(neg, vmin=-0.2, vmax=0.6) assert cmap_params['vmin'] == -0.2 assert cmap_params['vmax'] == 0.6 - assert cmap_params['cmap'].name == "viridis" + assert cmap_params['cmap'] == "viridis" @requires_matplotlib diff --git a/xarray/tests/test_ufuncs.py b/xarray/tests/test_ufuncs.py index 91ec1142950..195bb36e36e 100644 --- a/xarray/tests/test_ufuncs.py +++ b/xarray/tests/test_ufuncs.py @@ -1,6 +1,5 @@ from __future__ import absolute_import, division, print_function -from distutils.version import LooseVersion import pickle import numpy as np @@ -11,12 +10,7 @@ from . import ( assert_array_equal, assert_identical as assert_identical_, mock, - raises_regex, -) - - -requires_numpy113 = pytest.mark.skipif(LooseVersion(np.__version__) < '1.13', - reason='numpy 1.13 or newer required') + raises_regex, requires_np113) def assert_identical(a, b): @@ -27,7 +21,7 @@ def assert_identical(a, b): assert_array_equal(a, b) -@requires_numpy113 +@requires_np113 def test_unary(): args = [0, np.zeros(2), @@ -38,7 +32,7 @@ def test_unary(): assert_identical(a + 1, np.cos(a)) -@requires_numpy113 +@requires_np113 def test_binary(): args = [0, np.zeros(2), @@ -53,7 +47,7 @@ def test_binary(): assert_identical(t2 + 1, np.maximum(t2 + 1, t1)) -@requires_numpy113 +@requires_np113 def test_binary_out(): args = [1, np.ones(2), @@ -66,7 +60,7 @@ def test_binary_out(): assert_identical(actual_exponent, arg) -@requires_numpy113 +@requires_np113 def test_groupby(): ds = xr.Dataset({'a': ('x', [0, 0, 0])}, {'c': ('x', [0, 0, 1])}) ds_grouped = ds.groupby('c') @@ -89,7 +83,7 @@ def test_groupby(): np.maximum(ds.a.variable, ds_grouped) -@requires_numpy113 +@requires_np113 def test_alignment(): ds1 = xr.Dataset({'a': ('x', [1, 2])}, {'x': [0, 1]}) ds2 = xr.Dataset({'a': ('x', [2, 3]), 'b': 4}, {'x': [1, 2]}) @@ -105,14 +99,14 @@ def test_alignment(): assert_identical_(actual, expected) -@requires_numpy113 +@requires_np113 def test_kwargs(): x = xr.DataArray(0) result = np.add(x, 1, dtype=np.float64) assert result.dtype == np.float64 -@requires_numpy113 +@requires_np113 def test_xarray_defers_to_unrecognized_type(): class Other(object): @@ -125,7 +119,7 @@ def __array_ufunc__(self, *args, **kwargs): assert np.sin(xarray_obj, out=other) == 'other' -@requires_numpy113 +@requires_np113 def test_xarray_handles_dask(): da = pytest.importorskip('dask.array') x = xr.DataArray(np.ones((2, 2)), dims=['x', 'y']) @@ -135,7 +129,7 @@ def test_xarray_handles_dask(): assert isinstance(result, xr.DataArray) -@requires_numpy113 +@requires_np113 def test_dask_defers_to_xarray(): da = pytest.importorskip('dask.array') x = xr.DataArray(np.ones((2, 2)), dims=['x', 'y']) @@ -145,14 +139,14 @@ def test_dask_defers_to_xarray(): assert isinstance(result, xr.DataArray) -@requires_numpy113 +@requires_np113 def test_gufunc_methods(): xarray_obj = xr.DataArray([1, 2, 3]) with raises_regex(NotImplementedError, 'reduce method'): np.add.reduce(xarray_obj, 1) -@requires_numpy113 +@requires_np113 def test_out(): xarray_obj = xr.DataArray([1, 2, 3]) @@ -166,7 +160,7 @@ def test_out(): assert_identical(other, np.array([1, 2, 3])) -@requires_numpy113 +@requires_np113 def test_gufuncs(): xarray_obj = xr.DataArray([1, 2, 3]) fake_gufunc = mock.Mock(signature='(n)->()', autospec=np.sin) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index c486a394ae6..290c7a6e308 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -1496,12 +1496,7 @@ def test_reduce_funcs(self): assert_identical(v.cumprod(axis=0), Variable('x', np.array([1, 1, 2, 6]))) assert_identical(v.var(), Variable([], 2.0 / 3)) - - if LooseVersion(np.__version__) < '1.9': - with pytest.raises(NotImplementedError): - v.median() - else: - assert_identical(v.median(), Variable([], 2)) + assert_identical(v.median(), Variable([], 2)) v = Variable('x', [True, False, False]) assert_identical(v.any(), Variable([], True)) @@ -1665,15 +1660,9 @@ def test_eq_all_dtypes(self): super(TestVariableWithDask, self).test_eq_all_dtypes() def test_getitem_fancy(self): - import dask - if LooseVersion(dask.__version__) <= LooseVersion('0.15.1'): - pytest.xfail("vindex from latest dask is required") super(TestVariableWithDask, self).test_getitem_fancy() def test_getitem_1d_fancy(self): - import dask - if LooseVersion(dask.__version__) <= LooseVersion('0.15.1'): - pytest.xfail("vindex from latest dask is required") super(TestVariableWithDask, self).test_getitem_1d_fancy() def test_getitem_with_mask_nd_indexer(self): From 64a7d1144c78eacbcd2401d0aa06e86f4047b0a7 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Fri, 13 Jul 2018 12:59:53 -0500 Subject: [PATCH 162/282] Add a square version of xarray's logo (#2279) This version works better in use cases. P.S. please find me or jhamman if you're at the SciPy2018 conference and would like a free xarray sticker! --- doc/_static/dataset-diagram-square-logo.png | Bin 0 -> 183796 bytes doc/_static/dataset-diagram-square-logo.tex | 277 ++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 doc/_static/dataset-diagram-square-logo.png create mode 100644 doc/_static/dataset-diagram-square-logo.tex diff --git a/doc/_static/dataset-diagram-square-logo.png b/doc/_static/dataset-diagram-square-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..d1eeda092c4a69c311d8f53d069a9e256a9294f1 GIT binary patch literal 183796 zcmYh@1yEGo|1j{SyQHMMC8ay1q`OfR5G0oF?nXeA4ry7CTpFaiJEa@xhW}lk-#hc( z&oB-PvV-T|bIv!8k?N}QSm>naAP@*kQ9)J{1cKKGK7??Qfmhf~znKGnAX-VONP$4V z;xQiHApxILTPSF%fIwc1AW%RE2y_R$6tD{dxpIL(`@r`Le*=Mt9Wxp=M1e0LzgL!* zeg1c$>0d7aub??97&wDKaIgM-I7jRO?+HS2QB;vbSw|+qCgl0{)mx%!apeREWJ>`UP=W95DHSExx}r3$uD2m$J1k1K}CH3?%TD$?KH$qyj*?s)pnJ=BLXxI zBbAD^ZNwE4%uEoPhpV*2FslsdY(Ut#%3j2&9kfZE<_I?Tju7j@ewB=|CH?ow*e?TI zZ|GiGOj{~>S_YXluIRYUIZqkU3+iK{%Zl-MQ5u6qNhV69Pfgs1L}*?B1#cjaEAYOE z&559m3c%oe;=itMP`A?B?CFbDZ+MeJs$&C+-g@5vg`c(~)O{EFTP`#jow(yxj+n?q zAJNrt_LE5wAL29RgZJN)_VhEATT7(UrtDp&JE^CFhr!#_KlS5n@lU?IvX>=`MfOz1 zq7?(ptmgss)0amMK5eO|w5DA$9ob0!L=R~GukJ18sRnaK5L_vX&q0V-6{5{&j%#AK zSw0o;Q@)+|evAuIb$RT9_nX)KCkUxujg6haXj6PM0O^8nIondhfqZ9lw~1yezgkYr zRIHVa+K1&heus|YtP1)dKr)0%kIg4h5mkRJX!MBo}q#ucJ90I|1O0M2xD~Ni?xc)ZE3e`bQbW5-vcy z@q3HAi=d$L&zBa<1<9J|?*mwuo`xi{{V5N7kNBIgenKeky${1f8cwf!S8*D%T@PFi zKpi*_YrTyQPD$z1J(1(@c{e|90RkU|?s4yNt^Hd^OU27_bPU+k*l?HyQbvbbf~c}LtxT@+b*P{5P+Jz>E(+Swh1AERN24R$CKiopp{~rPlqLMN zKXiy7TSrdDbi;AN2|6^5TZvmyFZrRVSnxvOv7>RZkqjQ01Q2-0iP8rb0M}ynWbm6w zvoz8sj}+SK=6iHUdMI`T$laePRW9QQO@91}@D;(_I;Wzd0>|k~20as;m&EMy+pdV0 z%)+`ZHM`7n5>@^mo>8SYq>V!A0)DtBtt$GVxbVRQ5N}4G$w|hKaH4$iL94e{(koyi zh=4DU3>>nr1DG!nHww6L>R%Tft1aV%^IS`VJ#)2>T9O3uH??oTU*{9nYs@X_2=%uC?OAEDsy>q>@z+C+T zAgJLB;067PLUW=ALb94m_P#ihX;Q_29$aSjOk~9|_kjM#SDWn6y0r!PO0q+*CQ4e{ zH#0Fm(M~5fih9*hPMgk}&O|C@ITDoze%YI?mzG#FEd+XG#eNR4*l|a;uC4C|;G?5I6kJ%@fM%%>5vdgmbTbk2* zks`~${C(SR;*nmEL+}Dy3~@{Mu6iOn!O)z&l1NhW%e$O+)P1d0@rRve%b0`BiqTF4 zEGu;8+@^4wh8q9`w|BkR+9vw0QqxEYPCWo*%*#uaOBE^cSMmdi^^e21Wye`cY6VOL zcI-=rP{OLhg`&A4$+@dBxLgG4z4A?gq*X*;1QTZy=Y%~TlR^Yx*G`ia*ZY}%kB`kf zl;|9UyXY?F(e>Sn>ohLq;8-x0e*KRu6)}vkk72}EoI&L5^8Ffq#!Rh^R&3QN^J^}f z6eA*-b#K0JzkU79=o|iUWCD-iNA&^fvC*l~u~E?Nj|`5YdvNB>3q2fWfaDX&j@*u1 zB~_(Er32{P?VSIdU*U81ocJ%X-PRf3@_W)v<$A;-|CvZmFy9-J{FuOIidi?)MYJpG zdyRXI2vJ;5>N8N6KS7(YA76y>fZP^sr8>u#=mD^=G9P$4*{ufQmw`4>%AdJ>kx940 z4o3H$U=o}aMu>Jy=el3)J)yK!_jF7d8q>z?u|X3G{m0}_+2Q{i)ER4)B|c#FgDj~S zUuxj%z19Kq{l5NT9}kTN6DPKqD#xqFzX`#Ku6ko2)w(u+JbxUX-JY7jg&NEysp#zv z=)-mm5tgRuGwe6)M=y~t2hik{=a%QP1A|&|82yaT;wnp_fnt0P)1GCmR>LGcZC3iE zc}c8rJAYQaWyLk6hp$Jc1Kv96(y<*ktO?6IL2qMwIIif@(LWhm$!($8+t`dxl0f#Y z;V>k|a!Si!Ec(+%A6s#JtmIxGLSW1S)pSJS@m}y+@RBa>+~pE(_nHLYV>*zm@H8V; z#j^@Nwy0_+uGj3$pC6i+eKlaOCY?I8Kej){{(8xBpfwZhR|nwlJ9TO%Kk04sU02FH zd1x5{m}ffxaJDC55^l`bgClW*s&wmh>xnjBpOM_YMJo1A=20R+@*J#SZ>u9eG+S(| z^T$e*g2DA?eo}v+!IfYEoa&u4pD7!AcI#kL?4iDIonyQD{mVGz#ksKSG&_K?61H!#fAEiF zL;7f7c3HgE0!+E=;QRF}8gIEGP6e|7)RUJwbfw6B*J7!nsZ$bzRAP_;+3(E3Uzn!5 zbgJm)6EOoJN)ROu1+k9+BsgD8fTUZ}@&^G4`vl+q0tGx4+K$?30O29{O3<5J+qZtd zNoO@Q8RIL!-1>*Z4+m|yu9S-7tGpI#9Jw~4N5c!Y55vDke~ku?6$6+R3`?%IqsHWR zX(dMzUT?;rtGlOtDf}TNB?$n{y|9a{ljVZE1|S^Jz;KD$c`W?p?R)3+jU4$Um-%0d z<=4q11DK9aU#^u`IL*t_4Fsw=rVc+Ge>g^D+0g1Kia~iqgh`m3?n-G3UBvwu!Gm}< zDg+4hZ39SrQ%eI#uMc;nM+!)J>Rm&QP1Xo+qkNE_(NwdcUizN;o-fb%O%`43@aJs3 zyhr6$Vwy=j@eM~~vPmTHyZk=3RZ)59wX&@iJang2^iJD=<%|O@;&);ONl?;b&a=~= zrzDslr4p3YJeCZyC~>>Od2HH#W5&J4C?0FNU$m^TxPjqT4x9CT zJU%er!u95iMu)-P^h6oW@9gL_B0j9mgRE>(iCb_mIjyuTYBJ{871U`{BNLrf#e5fP|z2c#TR;Pv49S*ZU7P{8;m4&GQw6nI&}aeD}GAkn&>V7wJu^PVhEK> z045+gUQ=A@7WKm6V0EU=cTxDX9;6wcylv)a>SzkOSAyZH5CDXCNks;~KnYwQ6LP^` zGr0ZfGZXK6ABP^N`-Ux_<|N&A)%YIy(5-3NKD;lR&Oh#fWPP+JJ5~Z?ARFd}6FeP* z?E*S&V8&x`d3$lSy2~caesGgSN~Qw`{lXiP&%#&lbz}UIjYxy=(s4} z=3^FI7M!(-F(2!T=Zka#esuUcDJNGhNNAQthYy~iXU>Av?wh!fVrIMbei@iZ`V&O)8RJ2|tydsi+Zw4&ll#03aKtIl1{A#G{Msr3ph{{n}errxGr znJe>NK=Hr%i|Yqv9Uv@VBn&`(l62jF&l&(K*0)?_82XGC(l7V_9UIKfYc(Dp>5kWzw6Ay5;_MfCW$Qi5W3pd@|B~mDC;6cfOuJ7%0PTe>!;I2T zqygK;S(gep&Xs-emYK#bzMgGGc}pG9p>Wd(B)G2A=n5j^+vvo1fvNuT?s z$W5+L)56BS>o2Xt5DT0wmytcp^!;If+b1J+LB{KXrPL=cgUe6cRNX#^rq5@&)*1nP!=S>R;#?o+)*%-uNZ?9L=sb%fx$T&9wh`GSgLpH_=@d$_3_{nM%e( zTtxBwTH8rjjJcY}2c8`pD=uDQ2NPCAl|7Y0N!PtK-QOG~59Q%LoqAOu)5;xh`i^e??&0*qy> zgb}<>OKzs($u>cF)bQ<>Zl{ag4#6$i04nWeZC9;n6?4U3KlC*vCG^DLEv=Yk;9BN5 z9_F=vLW$=5|P!{+CX=Z*?OHvLr}0{se)zZ}2lJNj%Tl0&>2ee$SY z&)J}@Hidsl{rtu+^Fi2L!mi?d1^o7Fn1q`*>tvA*0rrsqw7%@y@7a$~RUg;-7v-{S zeIgk<1I~}GAp&R8o1uLexKwxmxTu9rWsdiF(f{_btWmNrq=pFdyO;&}-|bvx&c4&K z@9B$4R2Ss9o?o(h2JV7!?&|lJQZ?Lm9~Kl)y_R^VcnNm@kerajyJ@TE(}Ihjnk0s6 z6g%~-BcbW}(r>a*gMY{Eh(C9N7>I! zw+sR>szfXW2J8`2WLU$su>(Ua(S~271;(WO;vrIE`l|XW6ZPk%^KyDKZLiKS0IRA< zs8Fb|Lu|3suB;Br(0}f(bA~F&3I^U|8wT|KA}^KdV1@1~LQGw>z*=dXV9v|;;Ie%7 zPnOqp$SOW||KpcChD(I3q|yGJoQuo}?gEQ|j~BV|R0plp91$4mH^v>#NTX8QVn$T+ zcc>x$1Wy__@(En5@3en2C#-t3p86mZkcp~!X2G|fP70qg1jR+=uQL;f>ED?D30@oX zcDTn(r^>?30$nrK3A&Q+c>~0^nL6jq0JVIHCs=Bv}2mpO_tuod7|Gi#Awg ze3*vK6YD8N6_VdI3zJ#Y-5L*F=0D}=HIzG?Jfua^VeitJ;#YL^<25xMAEU3Drmy<@ z`7?jc7NK|VGvh3M(jRK#K9R0ru~S+gKqa-ev9v+H_*-*bBag2~N*D>Ozlkg0j%?-t zR9pW?&%ZyMui`P*j+b^@0(5dR(0LQE&x-G=dKN-#fSkmG+P{p%8@WIBk!5!_&R5tr z@nuHqtT{P7O?)eI$UcN@77Xu4-P>v&3)mIz|0HzMNYc}UByXtYP%5sIoyOiEi(U?d zC1V9PCpUvVH*#|1r9Ml&lJy}!1$Xr(+-}bZ)VHE7RZtd`?yTl(f4*ic8kkuTbw0=3 zEFUyVt;ddviU!c+dC6Wz?3;T_oIeg-2ZEh4$Wf;9DK8)Nul;_R9fntZOYd~R4!1>` zMTZbdp1L_C-A43$R2fKX{|Yf`wBJJ)?N1U#lf{!ue%>eQ?FNI%l`58+ozBsX#ge=k zuQ8#tCW#OQT+Zm!?m=^rEKkvMK`vP(mPf(W)*o{QVBL(+37P^KrEtmP?@RGc67WDQ z>~`#yV+N5Zkw7!jodba42L`IDsCE`pLzbAXB7N0`Auh&yg3yIWoAfffcCnkgwS(Zm z&{tMMpB3jbs)UgAhsn2ZVBzY1jvzDk?^wD5d8PRFX{-h;XR_b1{RK(NOu_cW0Orrw z^W;@Ri2D6Xy6~UBm!7;G-WEv5 zscKBaJs2>Y(G76rF^nO~@3;**N7})t^u3Im7SWYSVj`6ozd^HZZaaPIx$iiK?3j!; zwOjI6lZ@S$*sq>^bRRcyLD`AFmDqkpU9ITfOPRzVeV)7N?U9h~>?IYS`OEzm(bMnz z0r~AmRxQ|~@LNU2($pxup5KA}aNuF(sfFVGNH2Mu5a#7d`L%tEd&lIE^RzSJw?X6W z_4@5YQKl+cBJyWGD~AzN!JDa`OS0Kud6r1_1`}g3{ByY>VqA(0envY?#Dec`X>RZ> zqWPlvQ=^EmfyeY|Q*@6iQ(XZ|8J)Z=TuI-%mGPZ?7!pBNkpsaaiRG9}hP%N5d1YWZ zfcv)`+(ZLY#)8dy>5fEXSp+l%%$C@)P~`vpi6QUJhn8<%-0i7nVa?#hu+nCJ$<|ok z@67lg)p;sDnD&c0YP$o%y)yHZ@|9?6HTTn|_;AOBa-1ct%vqET2F%b9%)S&i|K(fO z^_C-aaJO@}3zzv59!DvR9!tt^GgnOW!zb&^b703@&_@5gKD^H`y6#%rhmijlcUeV^ z5M%~EGF@jc`M*f>o&9WAU7JwD`N?cTeC&D}fYiE%(OL|0EsEq#L`J%U_YI`PB_HXvAQNyg5^3aRwcF#G$OSiU~`+{XoDdVlV zqW9TBo5TGrN#3h7GApy`gJw5f)41HSn1K(`x5Ve1zgNP%$&T$%!AeURP@x1a@pnGI zJ;s_YI9f0&-*s-oVr-kBZ`6#ErfvZK!}+?MM6q~DHs3M_LtljQU_+EzZ1_ymqS?K` zliRGbLx$}J2FSuVwmt_(RJ7wme$M#VGv2#AJP#+O7s@Z5y}tjE)VI-Roszh0 z7r|*pNU=)iPv;L_I=t;ZR~Bk@u(f`;zLp1uoD`V#Re?uc$JhJ<+qTqOKB-^C?R%+8 z0c@<>wy_Lhb%SPTIuul?Q;&QiSpQxyTeSwY{a9dCs<@XKwbbmdlpr+h`g<`gd1IBpSYy zr|~uP82q<+&9mb?i0a^qf{?r$A$4Y!_-LUED~k`8sK=(W%BZJVi^nn997 zx(|kz^Kd4gd(fY~(my(#Qmu6HgB$J6Y0!ycE?`Grz)z>oH7F9MqDxhL@{+31yO(KN zrs=$Gs!$V_yfzE|S4Vmy{CUo| zotEX(z%mvw0d$CW0T87Nbu$fJS@ok{$$h2e>e{uq;HC~A$aN|P{;3!O45Pm0F%UfJ zUKou#m_N@w{d%-OWyc1D{m9tN*d!n0g8>Pts3uLaO8zgnLuCU54_(y(%zb)pVopga zNq-`j!C{V)WeJGlI0cYXqc|ER_p7-|l!}&$qK`zwn7>|#;ivm1Gn~7vUinJRAghY} z0{;>qCC*Bf60aL@xvaKVbP@buI|48t9U2%KKs;xJQHW6nAbiZV&$Xxdo9Cm^x(htv zd5$$ah9p>?SGHHSZk23q-4Mym+rIO}o!rZTo0L;Fsn1$y%Y0ZDNrL-mn zh)XGS9|-g6ebn{|dQ~FQ0}gAcAEob*WN$8t)$+vQZmV{@N3Z>E(0s@?aq(!Sb3TN5 zhj}9lo>3IUX%^R%PLwK@wUb(x+mxfj1!DoERycv+T_+&c*LmES7iAV1rZRl7^XL!m zM3GH)sc*Zcjm9hVms4FS8T3@iIJyCi)ATCie|~p{hIfe10*IwEb4S#f=#NxZ(wm7d z6dOO7@bs}bMP+Elk=-{Hjq%e1#HImzAA4=FVZ?-Rs>cj0T2N1i*WKLJbQ4f(K8?%o2q7c=iWk7YxaLrq@UU5kPyee72G-0>L4;$>w;tvU&o=&1dK+z)ABYq%R^w@9dQSbxK zDIcd__7 zI>h60E`dR%NIDp$(v}WJ0Im@(RW4Of+2DRvWBXs&q*g{PKRE8>g7oc#d*J0qOuxqLeztjue|I!gbIV1NQ<})KILvnK3*z%YW}!-=#xw0?kQv#}vsIx89-<_n8z? zUHwDhyS`7vVQ2Gi&s}S6#sJ%Un>nZ2GqRtxkbS#eSY?6i3vNuTPb9Q_nUGL>CjU)5 z?cKDpNUi5}Ch7?cBT57a4|H4rvxe#%BJ`WuQMEuQWjo-K@<43Ho7%kUUDZ3DKEJB$ z!*Gdsi06SlYg$y{n(z()<*F10_k|W;p$(?K3sI&VN*MMXfSPF_(uwv-!f8j@xFmIE z-f9jYSklh44nKzh$IaWiH!;^u$quQg|m#?J`zy%uIe_NcFze5ME;xMLs$JxPrVLhJQQiCc80-v2`_v_5Bu?TA_PvV28!Gz7p$AJa7BX z>ysA??L%mBoCbx|2n(4Bzu{x`Ky_~lCk;&qF1*qks613&@`|XJtCy?T<;O&o(qQ|3 zU1PaD>PvF~s4Df4vFXpVtP;D1))eBb7vK)aSr3=;mn~qv5_mO1>mb4^sEM80$rhmS ziN-L3NZqCC?}tB%mHrm1=BuC{<=j+hPN|8+O}c)0$yMpp9A(gLO$KYxx<7c@0Js%$ z+x5*~aYp#i1H!3&@mB2bV%fpvy)F4qdWulo7XV6gom%QMaso1XG*M!Fzza)oP| zOPOTrE3>bh8_#rc`#w9q{&usqCjKVkxD8Ba2YnSZt*-!>T@?d>b{`-(ffer~Zr^9m zq4cgS&403Dn%f-N%^s&h_ug^qJNi32Sc(%xLi8FlFFnq~zv~{CciboYRy?Q3D#i*^ zeeWxD0$3s28jPIXboALhMnzU$=v{k<<4|42P5Ph3=~8%x;nNzwjAr_JxTc+HvI~qhod6+T_6DNUkNJg zS?!|tplLa8)fpvMN^FTCCLnJ<@5B~V z+SNE8kuy8%NAHJ|)3#tJ#8?lqApzbJVZA9}CM<9~9LzY(_&j&v{H7wtRNcBo#(0_P zHM1qItR4loOP4y3^?b;=mC9*jh48*B&^Wyka*rrOnWoe|A70`98FokAE%KAst9Q@4 zc|+&&s>3Lf4Tyq)%+}3qtO;!Q>p!sEm9mg^-GT-*NZn0WntmE=VBewjW><6gx`L@e z$F`qRtk-_tx)ZX47{SHx7VW$f=Pb%Yt{EP}50lX1zxZT#Ps5z!B)J8uG_B{a1#r(Moh?UG=Nxzbs z@0OmFo}fDw*7v5Y9+v%`Nn)cZAE8lk3>@>72X>8Be3g5My){V>wTO42(yVlH%8_IV$F>EOf5RN&_NaXs%Z@v;z9w`N# zXVUPa;m z(tkx1{bwP5w#S$ZVSf9NaGovaxS%n0-!$lE7hsK>l3X^RW8PrQk&3{Wi@!y;)ZDLI z9f1#SmvJgL#*lAYXz<#xnR8*>_!ZNeQ)z2|y>#r~QtVanZG=0Mrm;WbBi&c2F_&}Z zbLO13qWGfG@-`u?*r;|BpcCQvJPt$8qqp*QKUlt{Gu*ai!{xHB!yl-S#IwOV;Ux0^ zwRb!}GVW4YgH^>?1!a%Gu4PAFWBSe|4Fsg+$2#SDe6lM+WPA=&A!P>g_f(f|U##C$ zJN;ZrRq)hkW;R8{V|y?6>cQu9CtAV`amN})!H4!VvWsbLXtti34@iE(*E&YI^$(oe77pxEpifv5AXuXzfrKd`aUjw!yjK35= z!v~?#qF%+$3A|rLWj0Z7%yEI^om+ME?|gK$>FbwQ6^D0f@G{D1!`BP%q5-3Bx(J!5 zhG!Q(^F9{sIr{colFV93Zj{AV@VBBDwTV3*?Kwz|e68L(MCV$)r8V0OmM5f_5Wntp zmJmfb6F*@<$nA^;{F$29qRO}fZ8n{m+}b1QuM&45f#F2cw=5e-u38_o7PA+E{-x(h znO2=ov!We&b~lfcnxy<#+LF!gfIg5f?==-@I-G|9n106fR?Atg=DWow#V4p8#L^rg z#sOwV4-4zFMAY^Dc1~ntc^T=|dW4s9zUoTNEsGhDq<2i6=HAcO(W{(czbjxXu?1y> zCnztLt|#s#hE#l-ce=N;_j;+%DPM3!f6EdE^M|BzUus*Bdz;SR40mJoeF}%jvYwT9 z1$S`*z7g)tcd>WIBCN1J0*#LUw#WYdi))L>j|l93y4^bdI>twrV%+SX$d?64Np5wE zvNFABGpJJRUoE40vh-R}WFvVf(1aRAMmsH1A2chXT%UevWN`fxpFtRnA0%I7;HxQ{|0};NrznnzJqS`e{Tj z9cB36@jv;U@}2Y9Bh4P$atTzwVm=0BsoiwNyD~-6CnCyi(>f=TjF^f0-#&bdFeYeV zcEWqf{w8`H-pVrj*tb!jkNxL6tQ@B3R}wSJ$`>zMa@MMUkD^NiJ6nxrin$Z*6Ue$n z9D)kFVI1oQ=Q*sJ0*r^=6CohGt~}CfCg;8{eu;j|W45g<_}m}*hL}`pxm`HW6`tmY zr^0!{w&V81cV#P%z=v<~w=cQ=3T5Bt+F$l%I%E>a5&}9Lu2wCd`N?P(n8a->x|QXio4dQZ zgGE>=6vC~@t%ys(y*`gODm|vWJv&^1ASRYCmjB4Lx<4vqWxB!l7nAw^*P;BH)ota4 z8*3E3=6%jg*R(`LS@#O~4}p$xxgOP9o$izc1U8f5KGVy`H#YJ}uq`<2O_=t-RYDmx>+J3juo>%2snBNiqS zCi*Qls^bgUcTONr1)$tn%hb@2&QS~WVF61$V_u{+yj%y2x%)z<8&gOq?2=mV?@Eef zl1g2(8;PS3xk6X3uMqOqe5d*D#RtC+_kCA*A-;H5a&yGyCH4U9yYFFucCQ9gNCn8X ztbR!dypG6|T`%U~O=(kCHV?5BQSg(vecZg3s+R*D zp*(U8_4DG};#=mfUdArQuAq_G&80vDd7w$?zf_1f=))hwSdpq@ zH|A*8={r_410~$hl^i2I$3Dk453!1}in?1qZ96~u^`D4=`y^Z?G)=Xqq z8>3M};MUOD;NIX~6WUA6dXAUq&G$3?iDx3G=lGl~LF_a(hw! zTrJ$Wk>9*=mbPGZ>Hhg^}~E@PkGdG75>;aL`eXr#B| zfFryE!|l@$I0C<%ONJbxI?bkfJN$(EH1h=iFQO~@ubr?73~V-J>_%2pfz{?9+B7qJ z!?3sZ&6St%WAp_lF5k@TqM3t)$JehEY$C|g$+F6XWN8_hiV)MlDf^OT z&8EOV-ksNBCsJ7{R$B%V5PY-c*6l+xp?ksmc3q1??n%_``k|!ZBpL=00NF+0oun^y zpTt)7nVQ5%eMSGROxtY*&MwCsks`_NOm6(=I&Moo~{uHeo-;gOlv$(mi8F^xMqqKSd3Miq_ zE=2HCyG45CQIfkK`S|Ue|Fw&~#ToYB32h41F9;@N3rjh^nH6Dvr~ML30Q!*RkCvD^{CYX z%C-O8)7A*3-b%`VbS;b29M)xrf{t+sJe$}$Hn@NR0_(2f4J-iK+yr)1!L#BcK6}=StPH=`kBqcF;HjJ9)3w{2L&JH#HN;*a)3}uT7cch9gBaAH*^Ux$^9L08 z6!G$zdN%x6X;{m0{jKn`;c^22*j@0 z?QB7@lBymM-dEFCkf=-}F7c%Qz>Q0s5#G+1^l;y~y#CFLoRSCXLMd@Lp_`w{Y2rKJc%Sk*WgP;yebO(j5L}1-C`Q zg~NZBjC_yfA(K+jD95i=mE1x(D4J12vg*PKl^_U`71Jy|JsDa~t9H=q*14ltw<&hN zajbREfLMO47}3hM75o41hAp7@uCmfKU8PzI74}dz&Wt@Eoh6(X5%*QOBbB|0*JzV_ zL_YgCdYyIh$V?CGbws${IUZfkmbp@uH4k``1@ihob>t-`%^h%LE_c+s|hPW2;3?YVK4(ueahY{gcB81U>S%CDJf@T3~q# zKaOM+m=p!H@N}E0*H;T=qfzRq$9$c$2 z&gLc9YPIt<)5v_XJEki}mN{nBE@5#ESanVaCTR3 z0)-dCZ~WW+wp78FVeZDbKcdR0rcq3qJC!blgZ4_>8-W5@e*RUE8=BGtM+e>8stbSu|OLFic{mDJCE z-Wyz3uW^YCVZTNcC%O5smSkRT3&4PjC!u^Ts{5~tH7qaJ8Wpk(W8OCFSMg|y8i@%& zRF0h@E#Ye)lUtQrMcNb;QjIGE(K6`#h?;7PLtoIF2LzsWj^W5igluB;6Mx|hXl;Tt zK}2B5*E6&SezaPg_|sSpNS zw;G)&`VvTZI^);(F~BlQ0T#|J@<;+tU+@y~;6hIuj+_@)86gWjQ_&?TsM<289;WS3hkMZ@)BBb8qLQ_SGgo-*!Y`z=4u9yBxGs!i{Rb0*MDzlG_ ztH#>K))rjDow*mVUUXTmK?zjlQvOg7AODT}bGnaEi!!amcb@sQXwsFUoNoBVTB-<4$mtC;fyU9r&}!K&`$B0=TTYKLv>1CgDTK?w}pSjfwn-MAWGV? z!W|HPxJt1X=YmR3GACTgr_W{tkZJ!Bpa_>Fd6oM3@fm1X1v)JAgrOV_fI#0Eix>;c zkj-#ROcK7nX}su zR#o^e{regny;~b+;D>CWv~;0&CzD&7V00Yr=Zb6*!QJ%|ekKjZ+@lxxwKEsv z(C@rki&svGHZ_gUp!?%hnD7D{e9{ybSrdQS;A|f-b2kCGZNb$ z+n{2+OziA1)Pieb(Y1+}M6>(pT=rZxr)^=QG@td$fRS>qw{;!t0a4 zW}F}i{=K)t#piGKZ>CPEvt}Dy?bnPbH@I=>rdJ0sfakvL7qqcj=>C3Xd^SVubLm}9 zEh}W9WL-&2(}Pb5Xwd?$XH(cdG4@&XctXCVTiz60`*Rh@ZypM2WuVP_D>X7L2ct^7 zOd(D|IkOu@8$|=!*4d&5@_3+nFadO9c)O?}?QVfb@QZ=9u#};pRQqATJX|%9uCw`< z?r+=$pt@Bre-@DGOIc&<&}-L}sP(h)<+_->p1ei?x}}rx*(8@uWQoU-qDl#LSW4lq z^bRZL%-}YL#=hsirnxmf12R!LC%Ff?2XI6G5(?CsVQif#f7IuINJomP^0E$qypi1_ z5$hjR`xQX+k{mg_1MW;pB3e3IL8Dv855Wr1_tUD(stl#D7f78_q734}j5$mJ5O0fp zw8f*IZdtLErl22Az4w1rT7hRefdv+=HYAF`93r zfF5f{Q?5DxzB`%&l!#(t4bD8uE_s-M7e5i%I@;kIBsp^a)pJtvr!@B4e_;uUIJ^Ds zIe87w1OJ2pPI(DbLRe*IsZQ@AW@&cCsyh!sVIFV1-e^8uYm}`k0jFKaGL$ow|7cGS zv41zBzF|w6{}3k3rd6#~E!CXEHkkrvwCPg90D02to;E*${xvyy>6d+H9>xkWELq>z z4#n5gd|YmUO9m%R_&@Fa+W|h^TbscFRiAU{YjEvb72|@Eb(=Y0z91H$wMV=-{@N<$ zD?~#{Nm6PxdnfhT6R5UCldQ;7CgTp*|d< zI-!P~Hry-=Or@IhTj#gI2_`g6(r$qaJbXNSFEH&cmoJy4#DDS&UAr&G%2|xRjJoK) zdoMOueMojl)()k1k=lh<7*TO!Xp#i7%HsvvIfw}h87*8T)FsqtpMzoAce%Y{C@bA+ z0Z5(RVb_=^f#eLpSt3I7R$qCf6Yz}FOwiCk9l+eMgb`9_2nh0v1SlfygQsQoaxVfN z{yvc2^4W{9&MjxWR}K8}6#0mMAMB*yuHcU9q1(!A224q8+0e#s?%XG|bI+B}@17B8 zv!i`D&V;Oe{e8eb+!5KyfPhe^MB|X>7kUm+C*9YM4^tx*ZhlDVS%cQeR@sT)@!^xibXOO1XjN-qNP2U?<9Am&xuchlR4>vRRx6C_K@nZX&sYg4&H zZr_inZ=AJ)Sm0>06*JR>2@xBW8kO4J_wRtDf`1;=aG!!$zs2I4R}(b_kUTGrDpS(| z7p#;R1g2Go!%OkzaVA8he89bSN*EnDF-3$WBL8(-J!C?S;>+r%x^zTzjF~`@!?mlkRQeBKsq!R?M zt_@e}nz8Ooy_#0Aog`#P(09A(x%=Z?XL0#tV;8-EdDn0Bjzapij2X5s$rk~JYn5Y+ z##R{W{BJ`a*6>avJi7c{*?!ULc1d@*Bdrqppwy|sPJO{`$iY;Zy=`iVl z(2%}-h%9bb*}0lNmP1`Bc0^!(-dOQ5oR$$lwirgHuL z_;mu4Uo4gI*XgfBpamM?+l!TXzhZxFtjH`FbonpZ%Q{&mS0p#v-kJ=uHUaj|CO(9S-i3jUfW_U>PK@GR~>#kjF=0|0z@jtk0?hn$A{pQ(x@`H z6>ibDzJH5ZA%&&utOwX&ozt}Hc$D|^5N)JHhA0N=(Q{!?N)1vD?D*5aT($p-_(ML_ zOO8-4-lJRRea?wudCTz6;h$q4yDe_xx9Cb$Dq4!KXdNqF^|SROT$L&Yoc+$X_xTE( zpcY!1^V@hLyY9TYTRd>6z-cUURXfNE>spE2!5-a{`OLta(n@z90<_{cw7lw%hjDRj%K0JC=FK+QsFs(mXW`45rhM}7A|W?Uq7$JR=t zcLI*a?hpGngf`-z{pKFHx-w%#NZeo8`+?7+)6D{me=tI@6W$m$=%8ETF|A8pSi)hf zO$AFpp1^!Q*XuzLj7I5qNlU+%egl(@(_8De%(C?_Pj4eeX?BIq-=4qC3AbvtYNqN+ z;RM&}1IOUk3Oh2_RRbrPjufxxQC;7Nc0j-jY%1Hc-!7{xCkCQe+;?3`9ez3=6l+^T;Mx9r(K#9DoUxJ7_{b8El~Cb% z39Ci~rDF^dePu_q%6UCTg4!(>{Z@{#xFL5tERD?E)^K}isDYX)LFJ>?8ZB|dZus}> z=@zX+0+BzKDd{@)x`E9vtzupy8>dDmMJMIXtQ$t#vWE>Z)B&ie5;H#(p{rlhfECDt zCU)$_-Oq&_d$xgQX6gRRFZ~Qjxw^fERbLT}^4V7%ZYei0h2CN#Jgwn80SD!Zx>&&I zN@dBqEamXCVJ2j5b}ym+j*CnD#ja}Q-Zsk!-`hSseYwTxduOx!ly$P&eX{Nj<*s(I z+ZAqPl>%7%SWKl3>KlFgo3pz|*R7zZs(T^@Cz30~ zvm?eL2m*IY`u-MT9)Ka|~r?`z-J!f$~L)^oeJ^4?`3k^@mQ8e|m` zX$$|GHV-2oBUkm~AOM$h|A#94YRQ2Su-iquj|2Ws0b-!@q=6N3HjQEfZ@YC~3e{Y_ zh16+h>oo%%FwlVX%`XVX+lYbfgmyGQ=xP1mfiL^@eIek`-Wx)_S|A607;j5H`*QxN z{uC$P|0SNb&=AQsV#CaN(TS?q?)OdO z5>t^;k>OOK>lDL>^m{r{R>sU^$|DU zq0pfag@hsh0|BlI@34VK!RuA@S9h++COPy}ZfFjZ})Z!y6>h6n@e%YY?x& z><5!1*R|WR^W2@zwvd3;{nlw`#S2`qoSaWmlR<@*cB^&1i&s*Q{FjSU4hX|Ly(96N zIA=9hu~t})v9`u#po0&|FB$%iske@b@{9Mj>FyAuQIJvrk&dAR6r@X9N$HgCmQ+EM zMk$G*1cnAlk%pmRknSEjo<07~Iq&QBhikcj#gff^@BRJ6RtO&5aTlK}f8^vle*387BSYLihOzK1+3UU_k#>>bcFIvavypP2RpoC$jO66;r;4uEwPb+s@bFPQR1KmAX^ARMo5dOj%STV#THLz*$AhooU&9yfYKwh$ z$N6H~Si3#{gbdSTZ+#y9qBq$!?*SBagBUKpF#bUp2!s*t5-;#`Z!3cJYIMTy1BJb% zyEFtNn1qaT^S@;7zvGG*$PEn^E*2&DXz%m7c(6d984+6Sos0^p&BPW~6a}aPU)J1; zpD6Qskz--6uX!2Mj*E4QMT$kJx%x@=whY-_raE?4!rbu+$12Cr)?Y!tf@Bk2I>>QL zEZBLjIsZGeCTY9qxab65Tc4cHAV@dWppc>)&hoJeKkIW6qEk8(;YJcy$*otUKaWOV zXi!SO;2D1LGvuD$hvFV=$Rk99-=3>YiQVDrPo%8p3Yww0I4HZ?I;HeU9wY}NUu(#Vxwi6W~bn!~5I@dt+5!t?Ws}{Oi@YN&_&Xp8Gt5chwI=uk} zOM#2(QhFEJkwcYJnm-{|Zn}yneuM!Emdnws8|gp@d}oU*f57wRzhnS}>E>WRA(C|! z{zcE+BwQY;D|3x~jem`w_;ypH;pxAR7SfHSHfP-nZ6rOE3YtT2z6v*cIL{Qz;|`>sZ)X)H>WnSSP2I^M8CuVie^gjS%y%+1deQgT3xc!pwO<1YZckK=>I~(HBxl8dY-xD-35t70LH7O$}Y<|Of5GQN`B%8 zI~}z$wHg3&i7FI9=k9sM%$v>0Fii!{`wN>i6V0#_>`6WA(g6cv^3r>T&uT)>&c-tg zJ#k#*&rQymBnTw|!RJK#jQSTeF+6(^vcg(|C;HKv-xkgi8BKT3D1>fOhxxva&(&W0SHJ>EIrvRtC+ABtBP65AXu)v82Pm~Bn-=kZ63B07%* zcmxPTQc_hOP|Ek9EGr9R{f}G>7lYalwKkdrcG)=KcZZ?omulpCR(r4^C^#gI;qn$P z+WzE*HiPaP;s1O(HnZ_4dwLh*vv?=G9pn+ey)PaIJD5S21D6Bf!#(cZNGAWQ{Qa}S zBkc1^>lEt=2l}GJ`AHxHDbE605B~Y-$tH!$9n1{N$14FLL0srs82LflEaE*RLGxvC4rL8ib`-JC&C$6YI$>&fo7x|Z9 z_)s@#WRG)sQnK@ErcI;|K=d*66F_D{1|fq$U;cimagK7dBj4Bm(?WikU^fx?1UnQ0 zIB65;03jZQ08ZL*6hYfuzP*`2=$JT=@&63ZR|*3Fv*I@ZjC?K>X2ZaZ$GC1f(E<%* zT-FCwETw2$^3;0gFOSkzg~Lm9UKLmB2LFfv;IKVXHAw$So5bs3y4j3%9_yzdwFE5c zdmP|N&hrO=dC89=YMp*cxRSR6w-XR7Qo^81f(g2u>d1z28mi}RCYsHCpS?79T6|&Q z<r1!a`KJ<4c>6S4PCZP&u#D z3D4TZL}tnRDPk9>fwNOa|CFE$5Xk;J3&pkE_1s7udJ_gl5OWZA=;zMNc*(f!Imos2 z1v(zYv_>z!?i;o_1?TzOUXGoZ(dIGdF=M-iq_c2n11J0Z zvu6jL0JA{;tt-vZ;4ClTl5BJ(|`oZ@9{)%iH z?*|aa{)q#$O?LpWh-Oeo$0yxi=^bZwTZcR&nK_@yYX_95L z(6J790l1I3IN7<7YZ^yIDp!M2FRd4&}4sc#{R;C5~486TK=;91&dWtHa?62UAd8c zo`N(&>H@$5@(sPXJV7wc1d%)8)REMYbDv?GPTNk~q`$i?%qz){C6NFpWutRLBai)) z5w30Hp@wYXNrh=Zh)WuPUccJ_5b{^N-7{kEUpYxFFl>S8YVSU9^}-zhZBKgucn>u` zb1GAPB7l$ zx~yce;)OE)$Q$Vq8gu?>6&RXkny&%eP_668yOH8fZOB#_3y!fLodnv|TRz1WcFa$&&*h~+X#15j1foI?Hp$Pr5V80!~i2w?+zwe!& z!B}n-uHw$7}??;jrzR=AV7uF>VD&0TV_fHeR zsN=g8O{#Iwy1`q45i98-@m0H3JC?N`-jx)zvF|2z?|GaSZR6It>q88i{1&te`YygA zS-qgY9V_KBoHPTKuZiV7jH!*O&8mE~%PT*_E=VUKvs}=zy+?!>V1v2i-HEqXz6l1U zmOnrer{rfxGHS%p6-f99SSSuo$@QAx9dO{Uc-aIVIWdpoA1Ry!1Og*4Bl+LJv0{TV z?w-0#2DNx*;U1qH8XxDDt$>N^$0Cc22k~BbPjTolvT65vo1Va#x%;De$-vr851PsNG2MnP?p0iuK4 z_gP!2tE#I6C;Nn|`ib&swPLweYdUoDrL?l}U1J&UcQ4#7fAq=y*_!IqUn@I)ShDi! zPutJpE+}0C%iF+_o80T{^wEsb49xQ+zZX*Bw+RFnK1Sgy0GaXWlImA>tl!uGSQHWL zM9Q={?^TDn#LP0&Io8fKj7n*QJ70=4TduWb;J(&+7s1hv84h>lS<$Trl7rp5Z&UbD zsw!0lQ>^O(M^3AVAgN5pOh->Yp(Mq_7cW7CaejtlZYyI?3=yvN`m6X8Pw9PItsg(A zHvPU{*lm`vWWuR8-VenS+*qT6F&=v`K6Pq6EwCT-iA>(CW*;59AF5|K%L@Jfe#$#M z5fKWVGXR)CWyY zf@iF-(*{83l4bXw_!QRtg4QgIKny!l6nJ0g`Gn0L|LgEF1cHf&hOsuTo(_s~Qgfdz1~9aI*r?-N8s88qTnkL0AQX6{WH<5 z`W8LW`ATqJn$DQc*m}p3ur@rDt1Zdw1==5P6hnb|w=Zs~d!B>ZKW+5!+Olq)Hvw+~ z?pf?q?N#kzYUOW)Y+$f8B^_&yC};nfSlK$10YCJO>vS01_gIZCX%l^u{>D@@-s+TT zW!8HV0ER{7C4kHo6hLB;)Iu zrKn3!U`3x!WTy$ql3q)_mBJ_5mss}hAwAlm)Z2>|A;)Dm%rF&iiO`bynq?V))28)O zSw)%UaA}PA2fcLI28Ks)>jTtW91Jegs){2`X=0D!UzZh;`t?u2XRrz;v^(MK|8pKd z4WpJ2T1Flx@qfUBz*eHV>6YDA?$3)yJgmNDDtpQEgc?idJ_&GNB8jrkJ6toEzrN*i za;=-AB)wF!;@C~B_DcOo4i((0wz1~S#H7rJS+(~a2ZWp`C@D0}o)iVxu} zsr8pTyk^Z`cvs~PY8D+jXIL&{ws|XCf_&fU%^7#r6mVs~%ot?LCDapC@!31siP z(mj1rq4F^FS5>$*;h^k^P`6$D1dF}V8RlP7X;%vX+j`_y#8yP;ESvD}m%z z*@MVxfdoSAs~zWRJ($CG@Mg$sFiUv#P0O3nA{gZ2S#g*BL`?@yftS{oBpQ^V>XUdI z9@Z>71ebP|I@$YU+MX0neYYYl?|8NN&;`^a8L;d9)&XCAqH}A}!+Cy7Vn>2Fo)^fk z=(abu&2O9WpLaf2{1_kfSb^qy3_5W4f2_KKsZu$)xgO+uX*J?c(3#A)4 z2yS>UWB-($;0)ZJMqXIW@KX&=i0)ToE1ID{0(}B~yjvYS7Y|~V7DZs5;-eWG&P7l? zPF%5LJx{?Z9m zTd)~<_Y^Lf;N5q*)wA$=20%}>OaqXUAMDBeAo;h0qg$8(f#%nR^#INEklpkm$59Zy zgKoSWM;{gIa9(c?J!NbpeIex@xPC*nNdXrr;%6}Rvp=ZlloqP`O1y2sA%3l2D<=|$ z8I`!T;xj4idfRh-KZk7$8Mj4mlQ_hZBZHC7hXRd9zm335u=v$ZcfDG))*@98f9Chh z@8zBs^BcYe?frJbWpj-O|SAQffaOJzww zWgH*pKJTz>uG?a@p_icMv0GdHr;xsE}IcW>B2qD{$UM39{A|({A6u8 z{gWHZ;zX3`{ebWu-4D3;NL%wtX$nl!NnBY43(>ccM;b+9-{K1~p-wS-T+O(uypm1F z$C^=BY=@tE)Uw~b+u5F;a(0<#X1ra;a}|ALgMPJ)sHcu8$0MYA$&;%4lvp)OL~BKb zI;vjx)L_m|-3{su^?rO+adV1nDZ@SIGTtCK-rp+?c@a2ldK?)-k1b5#*t(%mpq4I$ z?t#>};FOdPpg(t5u5~`>=%l1k3-U|x^qxonqXVv9T7HFCFw(s0Jx~7I*KO^XS%;6T zAKx-4Gt@>RHro^{q7qh726%auevd}n88O8_2S>& z$?pUWsmXGLtA-%l0tU||ahI1Q29~6GA#m`D(5#Qy{U$Dw$ zNu%kRdY)!-LXjU<3-#5>s*_U)L$7)SM*E+-ZXNRyeHn>`m(rQ0l}@Y8U-x}MstD3$ zwCCS+3T%O(2`B5sXL^h5=fJj5L-20q@66u;q3Mct(-%MWfTc|m!II9Hd zB5E(1;iqQ5U-ZA|4+(N2-iUBF6aDjwB-VoM{xt_%hvXuY%b-IoLBRI<>N*ZSBSi!o zQSfp+7o(O~$lu(6uK{<sQz ziFNH;kLy8-kz6Rx&Jy=@1n3cJxmW5U4bDOq%}$r|Cz5O`FVd4l@HO=e#|_6rb8`NY zwliNPqaB|5LD-lmNQ2#82DDrSB#Mp*jR;X_K0(1!r{`L>Y?9ql2e-#z@+tbyQ z=c^oS3LR{gx-6p!ml(;G3nd|IvX^fsCuUujJCZt*%5Jbu@1Ef=^!r%pNswLrtzaF0 zmp5Zi1Q9&?`xRx?gI_O0FC|ZXoq;8~JnD6J;}7NnW?cI^_)?MIR#Vlkr@YoM6_k{V z_7xw+V{ejG#y-r4hlvuo{uO5s*Ob+5xv@PS!dh@Zap;Oqy=(O2L1{j(MrX3mOg6}u zf++A1Dtpw0NCw03&BEy74vu^k%i{G|w*YKt==@x zPL45xUClI774($$gF9y^d(f$$j8t^ z&z)7O`{0Ustc*fhRfCf2rf7y^7KaCz=hcUVA#4z=?ae&={JSU;&z>)GL}vx<<^vw| zFjYowqj!+W#WHuo^({-D+)3y1M)yWv5ZMKgkSp6|mH1N_)F!CKsbL!jQRK)6`RUap zJ_^$P`Aw(JhDXqa{h$5cE1{JsD`oUv0k&U*&&}EqBr?QJkHko{wX`Jqro<)R<-x30 zHN08(l7(~Zs{Y)=EY~{Knhxz5RdO-f;REP+6!aSpj8s%jRITj?$ybjjThcwVsWquW zg_)Gq?6!n32oJ3$A6EGfMKkK@;oTZYL1QPJ0#s#0(x2XeJ~YC&7P1QbbWNO8{qVZ* z_&$sLxy(6w;qt4ES3ms!o|*1q0%dLU?^^f9QUM2P0CKNveDv4X5kZZP{|s!r8iLqHEkOn z{IISRmPp=qKSVLeGKhWnBVNSo3jj-G&rM|un0ZIbB1%J&W4LK_a4vmc?O8VAb}gYu zG!m|JbTuBj+&qa+iZNRL#=O=Lvwp8$`gK_d7>~CK^^VVtGZvi+o(P^`Z%J*rjjy#P zTJ}7#_|LLVZI(f#2W*CIcb6De7_e)e7=A4!QYk>uIx-EQJO0!7yV0FY(xPRc#gO#S8`OFON55)4f_Xb=@;!S(`SH75yMiYHt z&0eVF2L033smzW6w7XN2n?P=pnZmUl*%YpO=7>(0eB^hDHMI67h!$D1fIG>iUrK6PDq%{1$nj)OH(J_sJ8>-SbvdYvY+3VXZe-FG$#KR=|6S=> z;TgQpzFM(bK{n2QsIW}jQ+(xrb$XDr72a@*gDU*7nz~ce1pOuc?*LT*9!CDxa5~Q@ zo>I|n%Hzv!2ALdhKFGEc)ZEb<0)Y+ zVeL4ho(xJ{$*{kn&EleH{qVQP*+P0jI2eHyfH_Wrf~=g+>~$-!xFpXHN<#lDtN&)C z0p)bg6(3>TAiI*@$7~}FT9Nq^`Wd&4uM2>)Oa=sk1F%6{*8C^ae%tMwi)je;_aDC~ zL7H9lpQRlLD>;6heNxB%-xxgv^bz#_01uy=7lC;Ys(}+G>Yx8(Gu2-L*a&oqb&25+ z8KRIr(R-HjpO(@&<0l>%X`0#`miz2^%*Q*yj*$RdKE;6#Zm5@~QJh8>r+MqtlQK&a<(~T2 zEw5Y9$=B`=D5ghxnTPPV;l0TGSg?%l^rv=ZCI~wFFCF0H3$jA*<;fY!kk{*bitYLB zcdMV+%CxQUT$XL$ZdtaEyCc#vSdH zbU<=I(%II`s=F_qOk(u7Zo@6gD7Bu|0~7dz-m9Vu{n(m8NJ8)U@WNr_hOnQ`TLL#w9@tH6_lcO;EVh|UJdO`K#Uc#x*xH+P zEV#YFu4dR|(xA$@SH6dNm0^_u2MkyNW@4`a=$~;=WJ3x8c>k$CXy{`)8%6HnA(2PS z1A(-aLX*Oi!a;9c<9-W1kQ@1sd;htjw5Ktk3B(tGCQ;gP}dZC8U0+!rGXCGUl2N# zwQv{z%J{1q#i{HgwDF3}BbkaFds!6(YNZcGFK$lB^!9)Wdgm4O&5sMS58Tii-|n{w zUfr`zpnB~8_Bt)NpnMKZ#|R0AIF&$=X_^$Mu+E+s-Ow2+X8F}p=2<`gHnGg-#Op-; z1}L#JlWWluA8>b7JoCjV8U9qJSn`IVMsx&M_$9ipQJ zg>2lxJY&?G1lZka@xqKwvOOIO(QTA%he0doe=yBxUjsDI<$D=_l@>a<%q-TJAwE@e z*pbp6#hrKgDXFReu;#8WE|I5tL0E~rb-7H&JbvF6qC+^q+Xz4qOBn~q=|37@D;*4N zvns!9UyC$4^jx(4SAu_joepTd^M<107b_@Ox_kItRj~Cr=8_y= z2y@aVLRbRPEc|V6$8Na3G(!(O;>uXx+Kr6omS#_Syvtzx*RW4?LlX`}TDy-EY-RWs zjQ>(=QM1!^)o!fKI!0w^CNp#O> zLVbeFd7oi0SpBCV?ky=OHsyu}Xm{$EBis0NjGDF?M<2}}*g}xr)c^lc-rjHH8cr8W zpzQzyBQ?-$)+Z<4n97~dB?oX_Q+)ZgMOp$OTXr3^99%IpEILH6Y=xVwxA_kMNEn{S zvnf(w4d8tt=ff=dR0?2S7!6Xxl)|1eJo((j6OZOv?F>NkbyEc3IWo+O!1(na7yv^4 zK5zdN!NH@xxGCk51tWcrLqqzQiReVQa=yn0Gxk;2L-RXq!pe@HXNis}Q3Or7)}+VX zf?FGsKlCbb`W5e&{;F*qp8(Jl)e6-J)!=MNqu^r@2H=Xy#wXtA4pQ>g1W+6@dq{2i zQP~qa0NQ@e0O%j0e@0VATR%lHJFeA-h%OFz0m);g>g&t;3ZblgH|06y8T?j1?zLcq z%!u%F1e^bn-r{?pBp-@`kb`F0nK*N=FXtX>`K|3O{Dw!q4=UMkVjs1aHYA+G%SoIZ z@j|X83T5q(H*dpOjQW_()_U+|&)OeCC{)^a#4^hD;nEwvf7hzU;61e zIv<|B*B87hZ3bK8A2bgjmj)^PGwqGyW9(CWYp4K?Dtu7llQ&MtWhzb@j$DSrk6esi z@glS$mS~=XW+gIw=lglia({6|j^d1`*|ojd{glt{M6tYe95Esru7}n`TS?A~{HTGA z6|}9Z6)kS!>KHxJVQvK1=YW=$pMvxJ6P_Ykb^|vpfh`BkYp~$7B-iV#Tit)@2$Yvd z!^BkCkw!1v#05D|2F5!;;aTn8O_(EX(#!W$+lw!BsWFjmLfe-xwc@Q>0+OR4ES!Ee zs{!H}ND^;dmDL|`2ZI!g2~PrT{4ol${THn~=w(5mt?;)hwX~8pwazmgXY6_m}w@PcC7lQ94l(S2fo3~++1ED*3j>E1_t`|F^HeL4jQ=AZat*B5SKS1C9} zaFK%oSX&c%6uE<7wj5w+qAt>8=F{e0{_-avY*Q`{kkc=*1>n0fJ8Rp=?0H-nv#p7~ z>igx1%3UDp_1b`nLNo6Dd2XAHw*dSLBnp;Mi&Pb0Qkv1&b0uhbkMQSEoZvKF;8Ofpv@R|0DQrH+fCh!Qcnblzzvg9yF0|D~@E z<(f#cJ6kGTfSkY#xC5wLejPNX`-mdVMic(^PA)XMF7`$PZBMDHx_?~OJ?*Fe+e?;p z$|KPu(d-5e_$Pv^WEgkWaOH^aT%-d!e#x#rZzOJz$p>Wmup#lH z3OrIgQiAtDE06)oZsFYY?k{DyTI&Yo$^6N@x}}|0`+fwOi|AYD5i>FcV@6{}v$S-c zHBk2(Nb0iEpi`a$8i;?8YUfnje)d#>EHai|q}JL+7vJ(c$R_CjO6YFJrxsFfGx`nK z0}fiAU2&jR>XX0sxMf}U1RyhF_EV2MA0LP zy2z)tMth~Df|t_r8_{ga9H8x>a7m;4JK33%nVA`jepY3CReWgB1f~h87o&yRl-5bd zt>OgGh`&9CrPc#Nn)1HqeP<8zqm|(FBGb=D5%jtdAkm}Yn#7=kzUjz29dn{SmIqUI zp($_4MD>W$4&DXGRI-b*isoP-fjBJrD=nAHuYTc2NI~m(HvqRtk^Xz!@X#vjmiCr5 zpV5rXVo637+s~W^xl>6pMLISd-4-#W<*rK~EfDo8sGvJJAe_~Cd%coqJ( zqO%{RJES`-j5J(Y?`09UHK|1B%NHJL*@93ii^0a`#wI%A_ixuqX6!`mPj8kw9V_Ue z1vE^Y5@;#S;)y;i)J1(Hw-2ubqVQA<5`Ut}n+tN)7&qTDl~Fdj9&8>&co#2>0>eD8 z3Ji|ns)17yHTa@iuJBAfgT>9;y9wy{i`gT0&4>-qiHO*Y;d;;w@~ZnQ`wa=(v) z4g*5gX{_ZwX2{k}Gk^F+5mEyC*x@Km;&5o_*b(O=Eo$# z<8?=WUWqY9jD%=#lV^hmK@N+>+o%_o zWg9AhB2 zF}IPeeBxl7u|nXDb%(0b8F#azpRu~sA=B`uWTk=oJWb&@LL`e9M|06 zRYc2WnX4lNSAA>K9sld3{|-#{PGGxJ#zZZ#=u2^&396(Dt`Y zFQlw>fLeTOrv%qkZ1gLM?4X_>=m_JDR$io-g?TM4#>MP+!w?DA0YtNV4fGB4?UJU5 zqj|}_ws=c%pPhAVSDJ=NY}rAkld~MqL9?RaG`de8ER5lV^(*gyX2R5n)UnBMZ(J^U zysb3Uu>EAOHn8Q|7p096#~!2ZXXw<<@iQMoYD_4$b>`Egnb;EuXSN(%#jWLRLW|^z zNP-)G?nnm#1 zI!$6^nY?fmt~sMg-$7&?4aGWBmGs%(TirgAC)U7peE9IDK611077g0kj(-1N=VV|8>_U!OPjZh+&%>&tnfX$xa>#v9NMA)R$dNJQNsx**lY0veMSWhqpyAE7;!xr!syJ~QzCf7RUmzZIf|4g@@*Yo^r!$KFdWj!$n&3b*A$Y|{89zH!V9bQeC^Cx}4 zO>7X|TQ)P1L;50Bn%7$N&$a%kMqbtgSDuV}Q4Fydq=doz6$^4XHvOezL42TBi1X zW&&nRQXlT2DoJ~s_U!lUNv8(vt!pUh25i5~@X@3iqG;8+N7EY~r0A2|4aU{+vl$w@ zTDzekK(sh)Xgi@4SKSstC)T3Nf2tCkkNWdqAIq1{n-0A@SA#D2)I00+dOv=>cqZ=z z_Wn=IpO{caLeJbCrO*d*X(SF}RCJ2+%u!NkmJO1CU`E%-S&euPUtnwv*U^_*jsksA z-Nh>s$wyQSM(5}CgA;8p4G)SIb!wxFhpd=|FKNG}<(H}%lFFp@SFGwUlo?`Q--k-k zV0NH6@AWG8C zr)jlCJ>sfcV4tNr(Vzrk2a0$Ycwihi+5Z~0@)q3_qx=k1x(R#|^*0mNJ z84*V>=r~4O^881!mY!VeW)K{`SjO=+p&?o@aT?0*ZN%+5;5iV}Z*txQkamRmYH}?s~xL&l*@Fd=3-0od`C@&7Q)1x#!@+ z%Xa6%iPARu1ttoXmKDQhBrt#=)TlryOCzH<<>7-?-s`}+^FJ@6#OXbSb{}@=C6qCl zGP;XZ3#U+W47irecxv;dQfW#*JbHh*gy}l}$D8y7YT#?L!wP$)>(_X8=3pB1rkJtt z9ojdXxxs7cm4|LLZZwHa5~%xazM~nii@lFum|vM)nZ+(JoxErpCuOmDfn0&7U?N3+ zKTjn6al4<$I?-F*Do^4%l;B|6gEMLa@{#TRBJ+|aS_!5bgI5Tgn6BDLy&`)^REy#wJT(0|iZ z1bYEe0rQVs;Oy@#wv{Z(qBlA>I`OpnFRC^NF$Rw;zu}OvZg_@7Y?2gxk4*=t0kqRu;C=G%E+3R{++VeUS>1gxDy;+)^U1>Jj({mj0Q z(@c(>FeEAnv`GSgFGdu_mmzWtC8xK?nHCQ)H@;q8oA1da3?4|Xd1bZ}RZ2~)c-TeQ z)1+lsoFsUZ`Ol#38*bN5I`PAB8vvMgUhfMU)oTmfex!ggIA(li{6YLd`L4s|%t>v| z#iO=bECfac%^Thh6SNt86wV1K(8l*Mc)fCV-`KqymOc{uZJUmKGagg!_|gBPKZSfF z*^TTOA=$NVajiiUwr!Gz8C8()H^9B}hzYNqdMrIFG|en53h7cIidZZWO{=5Ue$3bK zj8742bR?p0Q^t)APA>~|+{c&=0y$|RScCYPm;sSIN!py~*G-wZ9O$P;r)WLfunU8^ zyxowir;73SaYt+)1dvk{K|AhdK8zjdi>@U2>gnlyWU-W?k^rr<&AkL99m4v z19R1HOP=#-qZ^khtEy|ZaG>rxR+&rsyDf{y>Sz;@j=_4v4x2W^*7FB{`{Yf1vx>={ zxcci!)PhOvSB<(Oq4oM*bW_JoBz@7Azj;=%tIp$Pj#Yr{NC%Q~LM?1WKzZmYvFgL3MsCu&92AL9rQZ6DcomY-0|@a4*(Fl;8O|U} zx3RYZ*_WHm^}O?ey~d1gF-ve0JzwJ;>@AW>(n(V3WYeN6%@vxc7LJ?DGvC%d{+8}5 zCQM0fQn=i~{OF>DN3GV5wzzZLpSvjnB;yiUXMT%EfnG%mvo&rr1K79zHmY0iUti1PC0tM^FtEsMO9nrb0w)-uhAJGy@=0XM#XWCD&i&rb z{q5a*o^-^eFWfKHFGXBLO3XwjLACgP3D72DBW@#(=ab9yL-9g-j+71%@?Gm)!M19- z<5&N${&(dc5|~mo;~6y39ohC3n;G##Y4w6Y)mvr{(yu6c%4Nm7XhFs8-OMn7p^wuB zaj}q!tD5VrowH8e^QEz_AL_T!k+u@rm@iCA4(*ezWlr4?Yt0FWS{9Q%G@B2bIX==i zR^2|`K1wmqA7<#&VVFQ~bdY5%n+4-*hFQ4C&w@+bA3nr(TpD6SbdcJToP}$2Yjn69 z8z;>t&FTRri2^3KUb9NApKtNE-5?~o7$%UiFJ{oKFm^_D+A=NZVRX*eiU-^I?; zO))UG6kyUzWKWD9O>9z7VIa2Ow=b*M5K@!k@ z6>(0Cd7T>YvSk(117aazA%O+WLs7muFL(;Aqv5vJ=`Bjab}+R$l1-y&g}@Yh0kQQPaG|0UzCeFsHRbRb=Oy%>b7odP}TRg)}WyU zp*@$E(LS>8_j`Oul+8uSsIY$5;#+MkcEi#NI*7WOQA(v`JIE zR8;02N=4)K2u3%#y<_F{-S-jw&B8veMw^E%aj78H@t2QC_N^u|BV&4#(w`%VEmDF0 zH$F;qKLds?WYjN^BV{Q8jMGGR0ww)?oPq^o>6&iRr(=>HYRs?-QVmvaEZ0ys0PEXA z|3ZIGtTDVoa&=4!g$J=JwD>pouN5y6u&xsWlptZJ;z5o&wMB-1Z3l}b%TB^P(slqj zy&>F$MGu3{qz?QSFM|-u0leo;SQp;cjR8!1o<=AeH;wblzOP>d$j6D489s{;anGR0 z-f65knjXNsRlZ5N;tk%-@m*(4X1253aGVvGt$@Gzr z?KR%n&e6_iMFD2LF7U;o;J=l1_K$~nw^ERQ zq`=#9smSx)-u8UU1mn${vzWiUvc;nOZf{nDG&&34Xrn(^$M*@*3*o6++VnlTL<}Dq z`Z#Q07vE8t7JkdT%ux3eT=x#a(yNP$6lB6779)K@pM3u~Xl2!RiMI zpb-XbaG|Ym-japh=`hhNM;@3MvS8uG|E(Pe`1m#=X!Cv_%+-hx%4Bn5RV6&!bU_Lc zx}xQV73x+Q^h*yqXSrrAUN})7V^OHCXsh;ku8JRJBtG@Lo<+LlxJPO-zVHYVQmgT< z?Ej>`IirdAd~wI!ch02tG#IV0vZ#bx&T*m<>TS&W)_r6>cbLri@q$rn9mL(JY4tYs z)-L;U>!uzuzi0GbcbJMVGV?rtQ)24AP?36D8Z^vhASDOV-Wi0goG(Z3IjAPw4-S}RghBPZ8zVEg9GU{d4 zN8W9ZeZ9c2MY}~yXeT=n2E`BG94%5SmLjG|Wq+z{gA>4Wt$R_5bsZa^00}+Cto4lr zu&iaId%Dh}U%o?8IL{7KcBY9krm21G$T%5~0?quGRVmD|<$%)cl3)p*o2eC;e>$E}B?%Gs2M~buQ;@>0u-Y z!askr!_`kB6GIX)hED4)8!mmp-ljmTkObUftFgctbnHth&h{=S=%lTpt%B^_eCt!G zTF5~(xaWhQWK3QeX=LviSvqh9RRW+}hWRW^aHVSgXv`X(*TLopV&u`7goQgrOlClm%uguT84-;fG z*K!A*XXLooYRS zG>Kyw5yR-5sVU}PdI|qE-e+EdbOjQvxA~MaZDV@W>Nw~a!hCMwmu2&$Tc1k|D;@E* zy<2)Ob>?r|V~alo+`m@1dHwD~tvK|8)<0Hi;X-2VD1GpCe_{t~v|RRKcrOjNX;2Mw z?K!N!dm(EJKzQNkU)wx`i;>L{3uqPX;_u?e86Pvz@_p%m7m#b;p$n}5=_np*J#Hw~S#mW18NAgYkUlr+ zBG>^oHU+!;B!3xyiAxvP@w=I?h76C{*gnOU8e!f@fZc%A`gdErufUJVp%GbKTGm#S z@ZeF_PX^bIy!}m2Pf6Q!-YrGaL9mDoNT>O2aFga}MkI5^owFLU8W7b=+ZV&S&d9iW z#W~D#u6TPzp3dGaAW4RQ%Er#coTV;U>pB5(FaQe z*@w(C&~+Ak8NvhZK@q0(j$yrt?$x)u$qFfEU%hzd3l_~g9^IwBN4}0zn4Twhn?27= zwDldaQM-frG;-aE%8RaNbjW8S5^uWUc3V(V?|QIxUCYW7{sBJ0%Q zWz(y+X@6^uXRHuH^x!OaBNRBahNNi1O6s@sXmANNfPOAsNpyQ`?K1F2zrkgWXUinG zh@-d$I!!kn>*!YB|8{?G$8@;m-a2l7siaAxYG3^h_jC4O^8)j4=J(i|%TLCuYh{(* z*u1g1$9DC3|M@SDg^Af|?z8aBf;DfX82{Fd$#r%B+8Y0w%9@H&QS;6IMLXF?hC_`W zMY2<+*$Wub?5Gy{DPy&So!5k86$$S7cy&!QWmkph3!wa14;?hkBK=@_-&1}!=!-dj zletgm!MEWN8+@OSNbP3+-h8O}LcXlFu@24g(<;g%o2^G#j^|Xm{-PTb&s$^%YLo?d z`>*7jlPAvdx_D(m?}P@>9%CjM(`$<6J{kHm*E!e8gPwTTSyd}e$GoQn{$W!o{8(>- z#6+AHL$=toB!K+h1p6}ka&W3cOhxB{0HfZfxsHxbcnB=r$*np3cyIf017RDRy(ewlHq6S_+ z$-JVAw8OWk46Yx)ryE1dC{5TfFR!{7xO_K{TZeS5U3Cl(I}U^67qjxL%AZos$#^c1 zGocH4pF1_D4JG9-{6DJRIxNcX`vRrAq&wuJAP9&^Hz=S83^_YPIwf5R;JFb#JBHE(QCF{1m{YvF&~iAEg1WW+UJ))Fawka0`KLC;g|2* zl^JEQl0#yWBut3h!6u8iX0if@<#d~XH~3N5cm5YOG;?^VyYud+Kk8d=-Sa(ir{%Y8 z)bfPW@sfmu_^plbt|`m`Tvr}0@2jfexaPraqI2XuyjFTV=dkPOOZV!*xyDf@fo3}6 zfP)wvzsKv61QTMfFt8Uc{V!|3)sDJLH`!CgVac#T8X0`U9($XHT)}t&<0kj>5sKle zk@|%BFk;mapC89KGQT-xu$0O_yqGxmMPi{0rHeAcKNCM~8aQL5oC&Vd zMPElOqTemcL|w`O2ds(hNIL$SFUL*A8kTn{a_V4mPH15kQ9$7To|Ip4KQ`(&OpS|M1%4bo$b5g zgj5HDOa_wt2q)+-X{8dP|0jfvLBYw+~`9ZM#DEdQ8svu|9%5o6Usc7BWKH>u}a?!w~Eks`?c+1m>Nun_|PU7-<&Se z;0e)@=VNQ}q4wShPygQ^LVgngt+V*@FI6<3YyR970K19=RPJXF!M0%wc-MHAR7KGR z3n9D^w}hSBdr+v*&DR&{p_Wl7r%6VfV`nFy0(zsYV9?| zwJ9*sEy2RCZNcq6&`A9vf_*G~&@|s@)LLCNA&yOlyc9X{J+ga*6TfBfGf8R|FhC)m~yf-X|aX z9$Xt8IBTRB@EvHvz(G1MB5B?8DUGl6_6PMU4!oB}Hk1%O-%Z;`A31^*|j=LrYAxK}xuqk~J=OIw)*RtN^_$U*UB zH%fkCnXz4vEPM>H8Y3JS`a$_!eWwTRaqKq7jA9Gkb#VaQE-pBurHE?xfFGG>pm=_o zZ0HW{(K)F-oe1{kW9T%l)Llm{<-*i!hxnZlxTs#vyr9r zHc3o*9LjZHaOBf5Q8ZIG+BbJIq1c&YtaiSZm2ovRkn)D`iDAjarhnhE^548~V682(Bs{()H51*GW_)VJMNt| zKn&VR97@lUl=l@tJ`IU4K^dd@yB9XaR>kl(@FyT96AYgQX~%frxR@b|v=NW-vJRMK@C9(EI5k zVn<_)_UT5d6Los}O$WdrB}i35)%v_ea)Cjq=?i&kvy6dXC4HRnU0X_vo1!n-rZ|ZX z4$FE=;E2SMv66MW<0MCJUv2H$YFjuR87?zXP?yOrvT@l1_7DPJ2EM!Cq?j#pm}2L~ z*638}^4wT@zT{hw%jM2?eF)iM)aGx~;;-ULUo>&*?JWPZ{IB)LPtInRbJC2%&3&a9 zQ8|s{;krPfwkYSbD$Ab5_}0=(R3s0BuG#x2NBnvgoKr2LCxbI=S^NV&jS!(e zab@vf2|sc>$=lM9Z)r#2gV{)EObG&|uOQvK=-Jg-y#LBlP(kvdb=OO$f!An}z=3|f zeI8b#I>}SfQvHrd$)^jU^IL>fSQtz; zOgz%Q!^KF|`a8#&9onWhHN!6x;FQ3BLheeAH8(IfK1b_refsqDsdLGGQ5^~u*A}PX ztTD{zP7IUQI6PttkiW-o6x1|;*~F#t2C<`gDi?rgUvic+u=qXrQC3l|e=&Fw>x0D9 z4~Nuh=l!j$b;V(r289y6b6!%@tv5O&_nq!z>dv|*4}a2)nJv<7(yXfx`51g@vU|eI z;~hmIhveIvWhMBfA1&fAog980;#hu0ZKFN7-|wVtrV($vJYn~xz!8ah{NZuX$hGZQ zfhPIWB>MpGic{8C``}g-p-4w%L_DUApL0C(GbTH{It z@QVZ-H$SN`0HX6LGnfU;aXnv^KJF156uS7ZRgL{=Sye$5@tAV8oFRLu!d;=^#`)wW z&+9Pxw_FoS6H49PWiiz;;RiAJq~wfb_X1p&mpelc=9?CqIFrYz7itZ%%eHm4g^Y^w zzH1NH9;#?momQWQ6G7i@wvQ`LCVoQ*>dJ3}567g2@~`>^CmN@l83#HB&)DDIwaH{kh}5rK=sIB0u-PRx`tg)XbL%6 z&#Hr}>>cbKg4BExGjQEcrq4w}3Lc=pIzg)OM*?K$+&@~SUs@h{RMWQ zU~so|;LO155XG^9KV4c~FoCOIR9;PlqeB=?939fVT{KrT2ctD;O<2nh){SpIIDI## zt;?c|Pqb{jP}>v|$$ZIlA_}`Yl&=c5m49A~+;AC{N0oNW{D%Iw3H*x$wdAtC;|v$e zn_DaRDngw4rzBVmEa&s7F{zPGM~v>|-Q;JACg~pmul+%(k$F*hP`9I80d9@F$^1>N zZBtw@?y3apP^Xo8Gc2ahGDG;-{w!kEr?&aHE^qnOzW@*@S4}|#oTJ~Xmbi~>zYNc< zlbt^pNE}FvO^l`gMo+ME7}7*`!FWHW(q{gLI6d*T#3f|)bg{?K5LdM?n)kSN>Wj`< zQ16DT!Uex?r8HxYFgXvK4(ol}PKU@D0-o`c&}~Aq5Y~dO-~C0wtLN3hZNa|jNoh&! z{Cz*?ZS(1nXo7j(bz;g81ME9_bm zj;{cm(hoUHUG?EZ9`xP81FI}dH9~JW!LUCe0@9(%Ko#&ArPUc)!~CT19B_2K^ZbSC6=?r7JQ0DwuVPuRL@P`yLqP8w|=v+bF_;-4dHc= z5l)Ec%UZ#GP#j(y?!0t0Zfxi&fBwyu3y~i5q0?od)1MfTwM{m|5_E0gypkTxTi_`C zd>bUmeoJ*{oh8<68{-?p#8XM?%dv$^Aj>ey`-2Xy)=At++~Q~_l60Uomsn54er0an zmEaHBihT11Q0iyD$o~PW_eSY>BrLCg8s>P6J2YPVdf81pGE6AWo-hAJx|>=l#epscnn)s$*liCJcs}S#Zp~!P1kR{0kZVHpoXmFa z2ZZU#!|67Mr525gN7vLQH5c?_lql6> zERRigme9N3u0W&pyySd!)oIT^6|N4wB$Ar+8g-qz2R&cAzYab6Q&nbXp7q|q zad&o}!YJFuYV>s}YAyS`Xqf3Nn{ zFl81C5${sq!dq$>nBV+!Ex173ZSl+nd8`Moq@ej0brO-+2WGjNq#lsQiBgh(u{VP) z1~BHg43KTMu((m+_qb*HU-Zaqr^ z&)}R0#(19@IUig5o&>c+Zv+C8rNHVy2tblq;Qf7u9Q`ubEdkkOoniokwWzhIN{j}H z3Q5obgFI)jlAMm~rfpFH)CIMhv+#29!;Snc#$xUc0Z{vZ4Ac{#z)a@CcW9{)70PMki|evEA?kgophsjYwEPOX&S{_xGsqusV9*Du0L3gkG2d0T#!-*Y`r`(OVvW>EhN=bs({e7JH z5^BH4XY|jF9Y<@_OhK7bwllWa&Ff1(V)L0U!gEB$67?eWB8r`kzJ=1XJ)R}wKMa^Y zz?qsk1MuNWN&?w*rOEq|U8#n+{jfb=ltEQ^u*X2uKvZl~zsZCN9S?OxKS3(yl8vD8 zyE=@`F%@CT$*jB$8?YDz=KjYJpT^QY0*QU*Nzz$Zw0uh3q487TU9DZb4v}kLsub__=YHi2@ zZf4wN+6`&?J>YJQrRyCA{!>l^WEVddxc;RXj?5X0w)w(o(xaOFnxrENloJn%Lok$` zm7MXOqeY}=j&RJ29kLuaVzaiV$>=A<&(0eig)ma`zw17u_VCzda>bu*Q|MEw;^@mcP-p zX0c{b*=?@wnq~!MEM|rThIcm>*;N-4cDft-8~U4tv3`$OYy^hw>2;yHcyz)P-Ufny z@KQZ3e%P3L#2@cu)UVf%cW-IG`R6SbBDpBDh^6?oB*pIb6Z zGfT7c+4pA%`@&{3V3_jQ>^<)fJyL60^;bP(7NlEvHXr%Vr@C0;fh%-)QA9*+o>N95 zDyBAI58M?jgkw=1&Hd?4x(+(%t5VVmQfv{l=W)H4G*WB)b(x(Z?%Gt$RE&I>!&|;~ z0|hVjJ~zQZ=(OYOrw=jfj~`1D6W%f71p9$X{{rY)!jqrKuq zW1#*c(FBpA|7s5yR6=zxY_6Y~kQa~P(BuTeN|QEWo9q@w>OO)SFSr8@t5Q(53eR=? zwEQUHc-1owHP{~4KLV&q$P>U{b=7pK4t&J{Td3GCaqh|^ETAR($FFiTNN)4`GL}fgF@0g$$oer32!Va=9iY!~O{{40 zl%`)`l8j_A+Y$k8n|zga19Q0 z$EkwSK5iNBR1t}TW~VJhlAA2>_U$rtIs(&!F@^h#q9j(k98@Fw!^zW~W?p3%Ytltt z_T?RDemsc3y@qt1c4<2PW>M6p%jL&P zFWDVFJu{ng=`=Hc!mK9ccGc%1CfR;(iy(ZPRBJW25BoY*ra0!(Y;LJjwNur3>r2J; z;7P=&u@GCoA9Z!Tw!x1|oKq*C;*8A@NFI}PR7R}UvS0flDaRo(7(g?VCK3>1w1o5R zYCz9*JUPm6_@r#xTqQ~>aP3@De)oPfa>2CZz(4)BLiN%~Yn^Kf&zF$}E?{}#>P8D+!qUw$pU3mFJnwIq zZA#{~oG`VIK3}}(Zi*cjD&l&s`~KQzbAn^buWwVZBxKqF7c6;{ccCpXzI}chJ`$H0 z1ZAmjXub!0Hb*>4vyZ7gwH;H?DQKz3&zrsLCD>J==%2i)0S`6;9rg?_q&`msGkZnM z=9`I%((2x(g_CQ%*Sdhb2KGlnHVoa2@RbCbPyTH6Tt3`oLYDT3&x^BC((QU~d155o znsbE>jx{FQyvANTnjM?fLHZ5f8JfX_y#GLNQi|-4TkEdhtJ!rk`mc2H%C?358eq4k z_^+v4;H@dvy5{S5apdF15$(+3y>G=6+;|jM!C*a;H-sOVPQ*{dBWH%XYY*h`le-&u1nZgHshT4$tgqG`s&Aia)*|G=JV+Arknm{h zhzIfojie*#p&Ai5Qnh_Zta< zQpVy>#qXY2*<>93-#OKwa}l#4+cqRw??1C`2ns`lBcQtu9IrwAE5Vf~Yiwe-81_cK76V_zpR6CAfM-l` zzkg(JJ1xeap+dwVa3UnrYS;RtV~u#X6DtkP%qpslXFPDzyM&~cLm&cLe7dTW6$i*| zBx!ZLBw7X?o_FfOHsp_iJai8|={QWcpS7DMmh8#(OFt0aVsLKQ*{|EP$mfr!XAX~h z&Q^5!VLx`C!ru30Ju%yGaZ+gQRVFWK2(9P)$t;d|cV|%-(fwAoy=nTlkwM2F@gvxn z$*euBn=`LvNa*KRy;7G!<)bW1Cs~IMsg?y+1$Za#ctDtBJ4VQNOGMn^K4aK=d(;|G znQ(0UCqWz}`wCeZ80*A`o27e>;1picr`3glme;%>sV9M|H?(lC`|lKg;yp1b@0fgt zaTIZ(*zyG1Rbb2kPmWjz5MU7bh-;@kiY@cFZr@}JkFyQqBQlgr_H4Hco4^w8SvNnz zq}`M}V)1gaMRZ0n=_4-SGO?l$H{XwIib=YI^Nn+Ztm=LgKiYM=OFdGdcc1XxYmlW5 z!``N)YUP9AqU53`c#}i-#n?taXX3Yw-F=3hi{{4A;f116-6~g_%uI63i{=; zP|1tDq8X_EqPNxO)l=0|W(6=-?HK3+Pj>1s>3anS+e@=2_nck}{|Mf8%&r+`iev^} z6`W>62R8>ddF8WC$bO~~mt<7BnW0lC(~=ppkS(KdYe}w4D-vVpVO?DDpAbk=+4Jgzy$NBwM^+74NpDM@5-pSd6D$H^9H(D}LZ^CF$iK%ur8`~@Fd zVSQ0iND=nrI6YS+m6=@$AhuWx>=+*zryOHEDH}{GwlmD(qqH}V;_ak?i2#MKoHG?@ zMsz`pE}ST=t;?tRXyPrbv9k+@F3iFJ9FZ>^;PcYfC3;{_p*Vxpm;!9CBEV3cIUPJMZw@Co4E7(m`q zOb5uQnB5*)+^|e_Rm8OKFEgpzi(ytjyREvhlh+lI5tPA9-wtTK4sIB{i-=SG7^8&-c{khDP>wQh~S-*C_z=J)(81TJL62o)Z(Ki z;zdeK{bw&O>djCa;tQAlhkl#a;}gc_9%aNxGvfv8BA;tN_YYMNnd6uIZVd0OA^etS z>*v)26*4c04KTL-Eea~RkpGfsnLj?0230l`E?|JIVmIXqBx(9@8hHIS;B5ef38j#T)#EgQNlC|Du|*u4 z986Gh&-__1AmQ3YDWW;LoxV0c@}-D$qKY#W~mLd>CN87G5 z!T4|Hthkbw--!^Up51j9H@-^-tw+*9Z>`@eya$j-rxfd<7{Kb#o0F8PX^ecuSXp5J za$i7fbj3e~UrsKkq(1T?=N@Qdn2~(ki=wT`zP4yZ1XNN`faY1JvVZs0sQ7M4{c`_u zS^{$5c&x1^*0k13=;7RDUxOe4wRfd%*Pv^N(i)gBxY+5=eZ-7z^~iUWLL z>P?c|#Y10+FOX|e={`UZh4Oupq4g)B|1qM&xZdv9sj%4dGh}_PX({wfk;pRPq>s<;a6%kkfs(LLB$`=V^AaP_hD#`v?2$(R=l(oJT`qB5~`EcW-rxB&_<0bU#=*D$gwd=AjYD;+&H%BSBB+aFj24KY3L5rndLHI>k@WCUlI96t~^PyxUfZyPuH`cp2Nvh zPfw3FXSYmwCOb>Z)Quoua$=b4y9Ndzw?MP2z99;I*_D9IYHI#w91e!wC+f!i^r!L; zr9N=Nyu&gsLi745m_sCY4<->_?I+`o*fN<%&mK|t(149~Aorfnv$8R>k?_o)RBqe+ zj}&GAdQhI3Li7MbLrmoiPV$k!O1hcPS#9Vco-AE-^Ok!1^H z)oE}Db-*bd?*(*ze4gZo#LaR6NH@3}w~@+JVAl!eni&)6zT~k}9w!qEw0BBsG!Dpgp|OP+<>%dbz&;_?55fTh0u}=2 z&lep8n2-ULu{_L(G+LlWTmOge?;XE6m#X+~qLa@<^Qw~?@)%9PbtU3ovmwb{gRoUW zQG93+Xmas02eCM0J7k~xi`4@%%1$0?(1G?+OqgujM<WWZ0r|E+Sks2(60HYJU{0rDCbRs`NgP{YI)`S)oGp1$DSS!JZ?Sg7@ zEIlxJJ}Z{Q$XweTR`<|L3BujM)s*aMM0M-}@1@1b%kJ)1J~A8La4*y6H6I*791?}L z`np5s59n0+vGS%`$^MCne+H&DSpHpI)F7aT)kW!5W$p2S?Z9^3VyQ8j>WUS^`!k^a&;P51RV)X7*v+lAzUDlnpl7WTwG{}R%I;N8!cVX> zdd81MH#rzkYZL~oYhU*NNHU`gjQ8PnFXphq6pb*8F}f$?b%gRC#!DMREKpx4TulAG zo;`$310IZxB=#eK0)KB+Paa^&N+j~tL3_xG!ivWhg_U;o;Xx$|8W9=5P5IkQFzxVK z^apnRw~ogR3occ_RIx^mX3vx4EumPK|MIo9xCrdobfpmzp+%ntJ`Z4<=rHhLi@}oC zAE|3G4{Wt4q5%Q1u-}$#eVU@-Sau%?%E6@2m*nB(!Bzk((<#$ox2tfHPWJk(E;mjY z0V@kV>x#|$SN!YLe1|J%3a3TjQp>CnxrE7e`uK|bigRzmX-)Z9@9#!AQ7;hafM?vQ z*Mg3HO6G~_$Jb1q08a7?I^Rd3_3OZ^fmg9Njne%Ycr;B!(hNS7ZP?~?9sF2TUEiGg z@QJ!P@CkBQk1>YUHw|j0fW~Q35H|a5#+K?-61%yEmY0?oj*>EiCSGXq$CJ+|*urU} zX?W44X)AJRe-(bAXQ#1bf6EMKG6l=8)U=j(n29o7blr4AB4=|>zs^ULuN5@D*;e~t z&sMT54A-CeTFnKmx)8o*oZkg{JsA<E#7 z_vATovFP7z+!zc$3F7 z7Y+?)}d>XwlQ;c_qJovEDPBa69Az`4{^BGVhsRFjVhyM46Jk2Vg!h zUpKN}rdx)fC!^RW3yu3>i_%o2=`q}-y2~L3pa>HKi0_V1-lUGMG71_TH9#~#6uXo; z#*>d?3oiKkT7*o$^>gwg-x|t9YwY_h{`isq5JC9oxObp`AS@E~QsX~qOeo*?Pv1YW zyF0kcxnscJ=EM;($KR>*LMbL9?$FuySBESMP(DqD9t`u(e|z&rB97kzY~qYV4%3F& zm~Nwv0H_A>#rD6+dA$hKTac!0kpVHak4<>5>67n~I-E3-j(`AxxU=21WSUqc)n!tQ zL@z!4`T9C4maJ_DLIe(K_tU;*kOwP7y90NGzqM;hA;P z&e6NVPW236!CxGw(FCxFzI}Pk3|%^jJy^m9H2L?RI10-;UJgvY-P+4KZ0O~F`l&?> z{##cH^bzC6nZ^3nZ`NZLPcu_ga%$_p=UX4`s3!=I%_!!?~-Nq*cyjlGg+fthE$?mI@@vWlR z{SVi^m!}l(pObxH#>h(j9{A~oa7-0o^qpfn9xMH=@hcdK{^g-MzKT1zPPZf)#teu_ zU`O6mvsSahJV#;zWSrZlX8h_#m!|TT)w^12-%eRp^tL5(Ds!==lSG1@3TmM5>_k46 z%yP$Qg&`;g<`b`tPTZVbTUR1`8A`>MRa(Aaz@mxtT4lUKnsr{l3B-1q6D8BsRhrc-<3l|B`TWOXlfLx7g6&rPW-u0h-~M zg!!ZN%!WOem@%K-`*G0Z)##To&iusylhe^Aea}K4-A1B_hZ*zUK3r*!zS6|g;G|x# zc<*$(EuM__9=oy?UJ43LO1bXzVOz5X^?86F8YN?-TC?Q4=TOF9!IanU$+#Pziu_<2 zU0VNHF*D$Zed;;(Lj=_0*XZiColpf<_z69$Szf?H&3|}yANmsz9%O>&QC`(V9$LEy zPx-F(h~xG}eP5|PFZSmqj?#!@Zh{g&Oids~7Mpo|`E^T!-P2FrOrG7IF(7C4OA1W+Om7Ym?D7kVV{Mf_ z?VtHctlf^)fBgSkS-+>Gu zJ_$YJ`ih}CtCMo{X`OnQ(B$eky4TYPZyDe4k>Mk=Dkpp6@_x|)hBdR^{OtBzU zpC9(#JEEFPayP}QD=+?D0Pyr1tF&#~%G$2lu1hsF(Hc@R_ri@f%FB9{oNq0p`35G? zP49Jt5Uu03%0kgrEoFFX9_cfx$QLEFP+&uE`N`A$t{a4i|F6BwTj@+v?TwU zYOjcB51E;Ew;d&X#dE1QSM7bLuqos%GKW0gGA%P(2~Gf!Xg%%b3Y=L6a%W*P# zXMq6VuW}5MhMDQB7o{1*mi@P(GxEqe&~S^#L;l>Ol?^|Tgl7affux%(n%O)FlIO2A zfzME3_~)^L$mMQ=>^-ZQqPd*BeA{P&M*Tbe@m@`k+EYwojJae(hcU!*3N~!L0t*(2 zmR*)zy3@VHPhI)b@{_Oh4h&o7e1bFB|G!M)_YqVcTKmZb{aVza8I#%AGuz;smjWnN ze|FG5xEW#Es{ef)E)2Vm0@TY-E*LH&{b34k+8O->S`dyTB6uPDRW99PIEXC9@8Ll% z%@_LqB$egWSm)EEH|*vJL7K=29X1rc&a+cDS^66KO!~|qvWc%p{{dw$`1^rL!G|dt zmf8vDU)WktO2ykdrK9Aq03r0#9*uM8us#d8OTHiATP9k^r@8IoaTDvgYW?^9CPadx z%$}eT5vX=p(-Z4l21ws7*M*+j!^A=3g&0PbNHUlRV&7JZ-a)MGUTL}zBSVy2$?)Zo z44cGYfKvQD+)m{?Nc3Dg{Hct)f^;j&8qtIx$IW&%Sg8z-biZso@y%+hER}xi3p?mq z)$8KTTIo#}W7;yDnS8!r~!t(%5utwMI2ZG0~N; zbn|0H=kNU?zzSFa2jG%ijAkxK))br0S>^3xx%Ilw1I8S$166K^Sl^T5w6AzJFI>ZV z=$+`Df(AT+QDWfvwYyZ}*5d9VQ-wmX<2q=Hueb_oEPuD*BD*=syYpfdT5M3@X#@Yq zEo}>k-nS6iKW`wH)92TKdVf5@992|QC(S0XSpi9%_?Wj-K7kgI%~;9hletY8S=AHR zu5A*yQk`q^Zo)pM@yB&8`)nMDY!Y&c-Mst+X>u!ycyxxIuuSx@pa=Y=G5_Jg8-_+vUqL`GPTdW0&~QS5%F+Dgmjr}E)1;JE%%D|nZc zQUtDUh+QaDf%ssH9e1vi?B*rX^Q`1FouT|h#yCr@WxVF#;AsaxM?W(A3JK9hXb*FLEOiXuBO_@#Cr6rRA-se3#W%?k z+KSV8AE~Ig6mn%Wg|lrH&W0wZ$|~e>8TuowU@>K%bX{xjG=`#p=KKzXb4W?CfZcTL zP>e(m*Y3;=8SMIt>@epPzbk$Z+(l~HblRE=ml|uJwD(P%0Y0oru;%$%^TzttIthc} z)|V3%f(D9nzdwf*b$Pmndj0Z^pQyN;Tc#CufvzAbWYM#XCmBHvYZWQh!V#7CKKfH$ z%H`5(ZfI^$-008#LH>fhBB@!zYYZ7YBiSWwrvujW6Y@U@846fK)FRSt@o@##p7q-h zG04vJPWH;R5OeH?HQ+Bn-g$QI;8hKNGw;KUJGP!C(P}t+_FMy_Qh%=4G2%+Og4Y5Z z6v=s_exgpH|LCdU(|Zb@HMX+|LHHM=-;|}9VUKZ00R8IiU%!-wn`9ns`RSiOhd%;r zTMRr1jzJK}@2EL@A0(2Urb#y0=G7VUw@D)uxzSDX7yisP6y9krhiBiDov$$`Fek7t z!x}>xv2kWu;wa)UmW1EcbU0!|TJ)_RW5*q@UAQ$&U|Il7ledy}w$PvSUx*Z4nx`Xi zXf9Ml%T+hw7b^B6=K|UygCL(@M8#x+nde^qpR)~q>}a3L@6`4gBv-LJy#D?5qlQ9m zq9)gOb-6Pkm&4`N5S}L_?HC^Gk9>$IIwi(`S@Jr5+Z9-qsxyLp!#Zfmv-(EF@sm{P zA2IZo6l_H)WY-U~5Y0nQTu*`{!%n`Qe5G(U?M~(H@pH@M-&0cb8HRod{lEV1S+KDVa6!!}$CzHbZJ-bd6uhF0T&L714 zT2tGnFhMc-YPM8=iU<1o`a<2PJpGZIitrtmS=Su`Y4FV!!ti$}^7eWW7gMlp3=Se^=i3Xx&vM(sR!kL8mJ99cNAo}@So>5LcP=#ms{NNQ@Q7r3%GM@1 zEN=MboHoeb(q->{Y{FMrVO|Tu26{`L){qQnmGRIXxUx8~IB@Jb>tz=zQCul}y-bfbU(oF2IiCi!7{1uA1f>y0~CF-r{N-? zy*C4H;)C)^YdO;|tFq2g8@d}*;C0#ZRd}ct z{GyLNEB)DyZX$f~Kn)l)gO3g{Pbm8&E7PI*55ipq{!dL87%htr&UTw#tBe2wP4P)F zsMxM*9A%&w#A~lU$vh+?1RjMs<>_;m1T0j5ITft{MYn#50uws)r%LWl&y)k)F` zFc-ijnvHntLooy>E|N%JG~aE4X22L6o5im#jlVSrd4^hD6wY(TDC-$K?wmK4)L(Kd7} zXm7l%qpSlq5{2rtHRCef!=2v(dbtXm`Z2fz<2mQ=ku#)jWu1Tsq*%_q{wz~?+`lMZ zuMZN{&3GD70eCF}Jky0dsa6R4ZgSzpn_~_K_uttcV-(EN;Zd4Q9DWwFDM(>h4`Zv+ zlhQeK!_x+%n!AV4QU8GUZjFF8O~d&(ACZTaZ_P=Q7Y||QY@eT)r^BiXeMW9WwiNX) zrh^q5GI`c{hBx`0Yn{+b7MXI_1@R}w!`;Ol8u|HzYgJM_64K>T!M+keZS9p15BC02ICC3RV(Tg)kN_C^dR<8vYP8$yLar3Ds$QMjhj>Qm9|JlB zf?WUg*-_k@{7inVNH1;hJD%E_mS@-0yty2kOfz$=LbF(L&mXWSzPe^ zJkMEF9e9rTT$_1NG9x1LKi>zw4+yTr%l$082^z=HWqL7C#_Yw|`Wa(|F-A;YZ+)4>#+uVCQEDW*=k(L@(;N;WUh^0HC%|Eb|Bp~@zH__+KCPcg z>!-9_aa9g~b1i)>dK`BG8;TK(HmA>gk$gCiu{#!W7ayFVUdDjdgYf=oPGd=DA!ShB zO5aZB*n9q`{R_F`dhI%N7+by(fMhKut3qtMq}DYV;c%;zidxsqUi-KyoRNqBH6@-g z;^@T~Fji6Dm-!$`>q(;X#t>o%F{@zB>so4TQTuxfNC6G0BM(AK_JDJ;8Zm-+M(-PW zq2zCtpgyrNUoz-1CT5<7nVpZ$z6koHSpo;QoHTv0%?bW=KZFDpd;gOkVBRDgsonYL z$&MBqg1ptsXj`woRc~BHbfgvRWE;uZX|w!{-Y`luN>u&@fCwcfI~f331x(=lpNPXc zssccJGEZf&XbONSz)-=#hJZ1P^xO(Sbcj$2x`)x}&>7MhVm6zt4yQ6>7TmPDw{iCS zoL}Tq)Vn8@E+sAk^zAx;($e135MsKU{QP!qQ?jE@oh-TH`&c~!L=$|kU;t5JX6U%} z#N%uX;vi&imV>B9@PU#sumS?-1ebmWQxU_ITs~jXo>OE{2#H$p!_AfAzTj*PjUGJ% zxQ9p7L}RKvik5OWpf&6Pe0EOWd|xe3f-B*2u!R6Y<1Bh#FDAq%q3H;h%%^s_=i(MI zk)SBfE+DZvP$DB&*pid8~nl3oct#%hHfX^N(Z}oBj!kXV&BiV`kmhQ zdr?7jeTT1YHka@2{#n;wN@IY{GtDC{Jm+gfI5Q(wr>1#w8OfeCsh!_qxRBP1g7;!t z`&-{C%oksKt)?bBDU4)!p-{leMiO=_nvW z92NS(1z-MuGp_nO;Jx1KlRA$#;Lk;yp-&*3Qhk{-$3;;me(_~pS+3C%mmj}~0o((a z9=uK;^F>P!ecZXXb1$}NRA@wq9r1GWy69B#Y!camlmlTZ<=i8*9|8D+;y`Db7Xm)e zfJp}=X*ZMa+xakWIayC%PhX6HRcln&MgCEB!}4F%jrX1ECZTVw*w14{=xiGmITQ&3 z2}i!@yuXMuGr4+wp`(_u<}D^WEci>1kY6VL&~c!?R8s2t`2``!7sY}mgZ-?LkkRq{ z@suXbyY|}cvb`3$wLI1J_H;k8=IK0^kq#^vAf*sApWl#7b%47}ScvH;(~8oI-cB4v zS~Bzoi5FTo0(^T2o%cO{uyemhfB0v<^i#22sPX2Asd^xTSBk*-2>#{{yffd$`E8SV zhB>Q|$*RxXvBcIVja!au{Gm&)4-vpEOtCFbAz3r+Gf8WtXNI_uudoIiZ;zOdn#=DB zH9Ptj10-7h{iWAoeRNMvg9^fM7wG)R>kppVT8?2gYDBD4m`V)Z^=zfxZ2w;QoFqRn z4A?qi2q8}w-kcPddqNp)4Br`IReHvF#{BIqSVXJdaT)^Byd1ok1G`hBZW#xA0|*4)ue-=Z)H=Wa9w6i8{?bkM-z@BNZYVBOK6us)aY>DPoG zL`#l_x|N>%jatO-n7_$dit0TLB|1z0_qGF+Humb~BNARN7glaQ_5 zHqr!pv>%w+pLn?`n-WqKN+OM$`W1$Ve$#z1i*(VS^k&Mf>=ih8;<5Sqz_$ro4agVN zh*BFo?%1quHV+bm8@?z^IkAjqP!9rDD+hJkO=szB@b2aeIkTT;Kck^QShy7wtXWAl zY4dzTz^l};_bdSLK*DV}?b=BFOT#bWwgOOj53KfS=T9y$XVSJQr+{!RO|>T%V_F|@ zR4t+O*-kPXC~i@2aO&GnCSsBR6xcEHWseASeZ{LcHW&pHyj<2NQw|hmJ2Yu@rMhPK zg!hESFF-!&uLsGo-++nEL?=l4MSqUIfD_?+mF+HD(*=L3xrSZCh*H*>DzJ6Y3zS|V z)Swt=@6rTp=R{e8RHXd~YCAj|g{TEl0yN8x0Jh=-_##OFyK$wiqpn-H$Unu{#)IAU z8Z*b;RX2~5)?rif2Z&!~#cQD=V9gDsjsZQ9|@pq|~aP-AA zL1QB*;-wy8uXv}+%%!h9K8)%-yTuy-Pd@oApwdP?=er4~sd3ryA2|txKSlg1wu^WC z=}4W;bg#!VspKz1-+2zo9=H+Qe&f0b{c(dVVc-Lur5N(M0X~F}Xx>m%NH4L- zrEG3Hcd4ye)>G`$&n2@6#mx~37713z4N!|4TD4PGTPDheU3Mt=b+N+0@bST%I)>QO?32=T@_p~{Af z)Mi*_!In*65l8q!Zy4~13H*=_r|sO;LO#eV10V}oN>lh3OZo>m#OvcgT;_jgnzJJJ!peE}G7obrs;^yixn|M$yjBl(jA7uy8&=>&S;T0-;i?MB=~dRcSXj_H=;F| zgf2)Dva-QNK*;!uROlXg4llsD!w}jY{CGDsB(`*-CoHaK8u?DnZ9P;}|miHrZ6liQo@md~# zW!1F2+X92aY=>x?E-18z(Kqadp!w+63t8oh7R{!BUQ>Pmjwz5-sujT`~g)zaC9ql4_sXCk9q zz3^EFIIOCQndAbOf4=TdvZR4z19q2`%MvA_Rl>=-3a9800G;!SxoSDRzm~W$1T=7m zm4I<7us%b60I?{g)W*~$6C2#P$t;n57x&iCki@K(=WZ$-?pm;5fpCwGng0P_CfUv{0jpg6?w17f zZo}T$oO@~Mq3o4lGHC7px052)NVm4HfW*SCnB0=X;Kbsi@rFyjYj;!&7%nnR8BN zbZzh-!pH_vUU0cTJgPS!e8&b6i8`hVDgdXlv3L!A(^m1U@LXz!+a3LJ-p7;=m>1Y3 z(>M+uf)k=Yt;DJm<_mU|q@_l4SSiB+)Y(b@gOQ~H#ztDJnl>yrY+e}$7=o9pYeqk! zq}qHQi9Pd>PSQUq+b~aW&kN0G1mv8nlWc-|K;0u9sov!053-x7x~o4ums+R|A$UYZ zzf_8xK|)phDs(Dz06r?i#~gol?YckzKI6LO;bjg5L?%v% z0ee(aTH_T~M`w-l$gNGl3gGQUnf|(rDL*d@i1a61`WdaC1Cf~UQ4CA2w$prPK-gl2 z@HJY67(ohM>@l#SeE-|4OKCkM1zw*uxY?u@ZxMJ1yy*Kno~G|8qa6a*zb46uE9gC} zPPEbs%K7X#9g_kCL=e3k0d%>z`Djnxb>Z~k1p4ngq;|Pu<`-E^rJIAW_a6yx+0e5j zKGduMpV!T(gXVDd>cPPI;8NduqtnRI=%Wkir_zRh$q)<(hPY;V^S9fGI2FmJ-Veah zmns11&>d4XX$N?=m{= zyMQf2s^o^qX}SmTk$6vSlKZ2f4~W7)6_`l6SIH1wBL92=@#%|~xrx@_E8eC5xPDd% zEH_1uA(GtR$$6%Bo53soD`<-DKB{oJLKZf!`Mi{?Da|H+04>=r1~yDf@95tV;rw3Y zV}Y_^TdBFM`LxjQTDY(3aY#&?UU$&divV)}3teNpj0ZI0PN<9&>TT5ju{&)G=oBYv zOkFu|)PTk2Md9)4gjz$8il4y>ZG)ZZh0+Iqak%TI9zdY6sG>~Almt%rjuP5{lSxam z0zDMcUos!Q{Di@JMqMHE1l2hoeeLOo^G4j#wB#5v>zv$sf#r`8_ z|3ivYYRlK^Oj{;jC&!}5*(I8wgEg|oY(E>9AuklSIepENC=BmMTc2#9ZRB}=>v4=O zjo16(Pp!GOt#KWy`TmRg=nFKB87l2(C7557Lu<%gAB})2NGnJsDJ``?(?$A?UHWZDpzc$fjN$_hh2PJPbpVAgpz{5AO#e)r3!)JHe;@Jn|2*QM z73oiNLV|(&=ELe`=T1R0I%7I?QZqT)5K$Dv>`XJ+0hD$v0?nEeKs^h9QjUG}Ec@Sj z6S?*MPof?xyod~H1jU>_+m>lvLKY*rfK(azh|tR5rBT&g6PSf&0|h-#ceK#Xd@gcV zq(Y`!4{v}&R6%u7dYz_U1mO$~1XSHT`vOGV`F>TxA4?)B4GMu{{+|Hdh`aT{GmBV& z@VuM?a-Mfh?~5?1H4-(xG43GLxM4gh?;NKucze>l5uZM%ZUWX$?@XpJQIU-y+d4y+ z2GlsL-#N|d@y%sOY?W#W&}_p*Y1}Cp*V^P- zWOcAl0MRy3NjG+zc|0)g4lj)Csk)Zn*!T6`%(zi|zUCcdGI$aQLcnWV2Toc5zKiI@ zc=>QqHYoiYEG=FGwZeZP%X^*c&{Tp|%31Pqq(PRBLUKnVu3PG|#lC34kYI|p?hkGRAO6-<)ykWVU zXE|T^u(c(-$d!899$)R#)0fcV=;mhDer&VW@`ZdL!WbPF?F`vn+j1Kc&c`?Vr`FmK zf&L$9k_QZks*il_UOK-F3{N?0KWkSxtm=Fe^NhYA8FFNm)` zh~?6Lsb$jEH}S?WZ+Tq9>=GC{FZWyaw`A+MeT;ofI@{#d#1_i*iNe)0gudPCBSrOO zwBXZ$D>GzV98{=^I}2Z1!%~$4NCX6*rNjVMYJfugC3`xhd0m>5TLpG^_=6*^^v!hF zQ|X8Q0E%oY1sMgI&Iy|$vmz`~B3U20OR+ED5`Vb8JVqv3@%>adyDbK!?MXm#;bRT7 z6%$i}9!kv3dFH#=3q16(uoM+xp? zR7&w`xEgfZ*B{-tg?%S$_q>14k;?03z9M(jh{i=jWA)`AM0Jo)p>_6nGD1y!?XojX ztW#x@mf$T}SJa)$-H6JG1sPLuJ56?Ad$DqxwA<+-sWaRv^wp96;BRMXj2XixRp>V) zb2^$epo0TUqViKi1psE>FOm(j6C7_eTv`c%>lrJ%pgoE*Gh#Vjj;hL)IUrCi2LQT0 zAb#S`v0r4Vh-7Q}m)!kea$&=k2uD7xc`DNA(-f>)D;; zm9F=v2H?#LI?z-VFQ1lm4AWwj2b{apu~Y<+Ut^#8O(7~;CNn10OLLmPo0a}Hc^|Oy zWu$pTI{kIVm#Y*j~Yp?KN?&bK=tk4?MN= z)B&M1(E7Geq;O<&WM3Yc8Yj%AfW*n4XpF|J)}j_yA#8w9LWH{C#-7)X7e=F}zQgKb zcRto}Y7giP6JPg}V|A?koOZFH`q|Y5O{l`JkVu$fFDFCN>1z2jH*-~F5yJI_8;56n6dq!z?c4oG5@}PB$M3-Uf-F9dpyYxl+ z1koo66!7d=^BCI1LiR#8&rh+|9pai3sDp;Xlb=2o&SzUzxbEG!h+gHZ8C*o%<@_^R zAbX4gLMZtEE1f?wh6q7~CMXgsxU3n-t*zf%zn99{?(--mnrST8Y{=DL=v~&{dno{| z-LL}wqKHB^!Fwv)E%v1lklv@+kPGa1_!|6jfu;Vz@m`QB=0SS&+&A&eX=+%K%Mbl( zfxxHOy`GcEZg#(#uAkGbQem{ zm6lEVo7A6nA=8;ivuA``(9`k(jKAz*@=1QydI@bi0M^W15-Glvu_WsHZ@e*P z9LvV0g174W&O&w{3EH0rCXng)dq&z6>VilGTNw^&%HKtB^GU|P?xr;Ho89|vvyTZ` zJ726a?XcR*?(N{F3vis@e^pcSQp2NetXgNAzCgX#FA0=xIzc%_iAGgh2IdL}xU2qP zK#8Ix@qGRHx@7k=yi@7-s*CE8P6}7$k??B3gd9WTS)pXlLv*4Xl~W`N9wf=Sb=RtO z3c@L>7pp!4U(j7njYlDc+)VXr?$@@ZFVG)s53-v=iou3(14XCtO}(4|jvmcnO_CE$ zy@Ieh{WOj-pclA-Qds786e4&f+5FP{lB;#n*=#%|y<#|6yiwD!J#^qkw!o%TkY5>&XX-`y*ZhteyDd{4&BbltR~Gv2 z@Yv^vZ|(kJJr{5F_O707aBSfoFo1o^{t%MrQqhWep1h>?zL+Uow+U46bg^@_lj+iQ z{WFae=Um&mbgH@WaCfU5j@733_AGwqIOCXlKXEw`Wcgy9 zV?dWsWGV^O@%7r6t&a(=I!sB>4kfgr>=XbzB|pc%ZY4)^FXiqqi3~Ww8s`A2&RXV} z0VYo|n6a1x{&Xf-DPhLlL466yS3V}19X<3Fs161HvD=h%IA7U|9`bBj!T84d(rZ+^ zUwiK%4F(uiS8u89YpR4jv=unkjRfX=anabu_{MP+dQZb^2S~?<(?IN&h34M3D}ArW z@kUoPzHVRg&+3RZjhwHkbxifx4{f!*s(zXCb_RO|s|HzxEG(o=<#6-;Sl6c>Y}O9` zL#lCGf2CRHi?U^VVRPYCx5v2caWJx5Z@8t&(l{)2aXr55TU+maDttHs zzhAlz*&o{#ny>dxd7|57`Bg|`2J5@;-Z7YL6yvDA(~I+1O!3}sx5rXHsPVVzk4*jm z-xddOH`tH;nzNv~827UQhAp6jj@nks2HJ`|qw!N2l`r>GV*eHv8%Cdf?oWYdD!GXD zp~c)y7!ew!-85|Z*MbqGDT{nC2x z*RWA$vBW)2RH+L;b#|8OPeRg)iq-pGy_w&pa-_L{R}mtN>)t$oj>P}w6kjgbft1Yi zG};6E=&70a_fw`Uu^$uSf5gz` z+r;^jD@cl!c=p`&%^n(Sz*8GtBmu61AVChQX$NUi6Zmd;O^S^)dE4{0ZhX7Dj{fpVzdYDjcmb6a|ixheVz9A37r zGZd?>40UD`|9OSbg$k%dSQT0q`f+&peFPZPZ#6&Wu<;R_JG%w`;Hi35s7$C#7#xSh zuzl$U0^VR-@Lmw!2VmzbKkgqDk3shv{@#Vp-494WJtM1r`&c0Q>l}o_d@~!c5jxsZ zo4t09b)!>DTQ|K|7#%AkU+-mrFdQha-)8omeoO4*YrT^9-Lu{%i%U>{Ve|Z7(=Tbe z-Gq@HW1TD;NTtl&QGsnqz28aY<1|GtG@y6)rO`?h@u4Gh9oe6rKS?=BiAr_KU5D?s z;6Y?25PVAVN&MyVP{Imk7XkfhOdu1UQo3C(1vF3KjcRNZUvvpf3JVZ$L=6ZoCC^XvfFoqMcGYF& zNN{=VSg!Q&WsWd7AHIw|4-Y7V^ueEHrLXH}oDj~C{=1#;6C@8W2(U$f7o4H7cZG!( zm;Wh89oz2m;8u~F967~D{6YqZNRHC~utH7|v1#Bd$`oGOxPmeyDmoE8E4J>3uc%>u}T>@t->#z?S0eqN2T`z0pGqtc(%9J!4P$NzD88 zmuV!yD`qBJr8!JiT`~YvC&Eqp;}7RV=596KlIWybo(qNVjAXmaKNmDns!E!})LQE8 zd>>FxR+G}OjHclY-uRPQA1OMM{aZHu+0B7uX+K|f~|+D)oAY1LK$0#cIV`TgJ_*U1l&nA&3eev3|nBj z64|ZcxZx%F)G!g*AJZk=oe9G%cYSw^(fjk%5>K|iiH82&*h^`P9-x>~4%ze7`nM!S!7Ln3!|&&9T-bGn+` z+h}nmfR$>vgkg1Wgfy=(V>y@5QFtVntuLP!K861_zbXew1rnlOYT3S6UtmlV{WNSFi*4koSr~|a1Mm+@M>uwXogXWrR38ha z$tma+}fMHc_+kH3Sru(AL*_7irUU&ZJCX~5KstT4`e%~*L zVo#?aj4~5gt@R=2Lf~*G?#i)aRLv5;KnZlK)Ztt?-G0g|@cETVPOZj%oyWe^MbeOx z5+{bMxtM%Z3<=wvwmvyRW1pB93L`YcPP<$c! z>dCUlphB)ssgFi=Dkv5S5D4T3Q}!gY019PG_WQ2CU4Nse4DCmjC{?`nNyp=LZWViRIA=rKdsR9?yd*kda2Cz1HGtq44}ctZM^<!$;77O2fWDoDWJgy_7==VS%9E@zTM>BznEp`-AJ|hc`$v9tgwbMk z>#q~)GTae$W&eYA`O*RfQ^g1ReQg>y2@eT_n6-6JIY)+@-QAB%Tj}yGf&qR}ymsS% z>;}L2C^vMR5+Ry5Wna)k0sp$iA-@%yu37rkrocA5u#5xe;GP(eh3E?n>=WOj)y*@a z=Faq2w&!|SFDi4q(w^exG3Dv1cSv8i62e7p@WGalLcKzMq}c|=9tHFs$WE8L&8r?WT+qiOmOO7nx;JzGHEv zEiP@CQhF%A!ttm34WiHx@d_e9+Y7z8m^IG-VKa#4TPL6C}iic#snxzUwX zN&(!9_wB$A#p`umLji&z|Mht0NP4n#Gw%sqRJJ_><6P&Ewp8LK;SyQM(*6}Nacuw= zZ5fyTm%&w=bLHGqU?Vu4jzyp?w=!$)aqdT`g?V0fBN&rsLijge&O!I~Ka=eCb(}A2 z$u#fW*-L%;uP)CQ@|Oxi(inaG2TwUX%XctXw;?H7&S>OC0-Nt8`?j6%w(@x5T*^Sr z1(ERCuEKA+GH)9Kb?@jr?`fX4P4e$8ukWbIqT;<|QhU;bcoeQ@Aop^@NICrLtn9ZW z*@)!aH#;1E+WqTLnB|Q)zy2}OP(hd4)Iw#aSIg?Xy*4s?zjS?dU7rRtOV>Lx?q7C` z6!J2ya@^h7*_l~~jyng-Nq2SD4jhSt`mEn8^>uK$Wz658SYCM94ZgVDZcI5CvEDR^ zuHf{x@KD+)e)jppAk=PSe{eoSBBcBi>;TmFv`f0Q?Ms>r4jM(4;mdDJi?1;hQY!)` z^=WUpZX73!Cl$Bcc3pIo^%l;`Cns>uq z`xa_KS+VVzDdlgP7)?-QE;^A%+Za?K_6;Qu>_5$960+N^DA(N?KiUF6YFGioamCfR z5k3%Z#2KpyE>`5=?#OP|(NVqlu=nqP<3vr*f-{s$d_!!D_V{iIG_XNl?my$k`!#Nc zO!A-PKPqKF0^V^=@7-GT-p3m4%yIS_d5&6%R~TGIi{`Lf^1&npCg(M94#U*V(*EUG|Wzz8bsnk>-3oFfZ8(3_Z_@U8sXTwmRB2OUOG%~(;*Rz=z8&pJ%3HP!Xw>F8N-Lc@PyZQ>_*#BvWJ=f`+=iMa;cI+wpJHSD2XzIC{W3chRV7aE*QqovIB<)L<*nn4r4!!=G4W561B^X~CT$|7UvD5P~%#zf(7 z<5_iIb)Uw^n9b5YGb%xW;wj5=G8O~!7L_(&&T#+loYB0F!?ocFNhe#f*gG7E^u;`L zoH^a|unV(1eFImP+$xCS$@4&^uY*;lyUIUAzkHy|pGO>{{?u_R*euFyw1AJYz&jQkMwz;q$hoiQwmpkD zXSN;pgFPishLhTPNqlhI)tWkD;U~1Bn~(!jAM`I~Idx=wxJ!VMlNWL)Dseo?io(W` zj9KCkk-d+vH!bL^!ik5C7(l$KTO+szE$ylC zsClEW&at|QJ>z0<@3>1syYEGc9zz5O)<+%i;&;a(tnGT`z3^kHjR9|m&XMf@7T{# z&Qa`YhAgb$f9T!KL1n0o?lBD9C+A7x4Xd`5@4<*kg)>z}tB0Qf~T@2;_Co2U6ZRXUknk`Jz9RAYR=#FBXUM?5ME zlzjMEDIl?5-!jC9vqz+iXdMfQ+P@M_Nh2zCq51K|9)F2X4vRKb<6{jzo9*9%ax1K~ zvBG0ZNo#EL^_#LeM*5gFB_ z4B?AsNML&=rY^5J3e29U1tPBKRzTX=zlQAFb6*qf!1zHT=%#rH#X0iIaEisQ>3 zeGPwxPlp@FcO|wGyZY6jA#L^yDZf9c_f#M9lQ&F@(fiM z_?_I`O5FdMyR-;3ygZC2DZa5s&oPE_Xz9GXxF(`pwwJ~*W|nl{TkzpzB~MjMRWxe9 zR_I0LWv*Eih~u47jq;FWd?AQRWGf%_l@`Q$+d>;8<{!HmicFR&F(wM|xJ<$G0{uqE zETW6XtmIG|8j_~x6Lxmq`az6@+<)^C5G-^4ygl)B#rac8E#Yt&+$)q7txvs6j}k}d z6qj@L+q?aF@z9=br4d@-C)wK3V-l4x z650ADi`bXg3r2-!!ic|1Rot<^cm;V3V1i2bop=b_> z6{uesWdq@@Gs}d4eUD=gs1m73qVpc5OLtmtnnsW*Av8Pb1C&Au15TCPq@>I{nQYam z>toa%0cg){u6~|WsSGikhs~JNyFY3-As^}B6bcxtCjftl(4SeVRH+m%3!#?u z1as~zAB|_BMyfzQ1S4yHA!FXwNtuAh$#LIqLq*R5yzMftH@i6UL#hC zN**bIt)H`P7CVviDrBkUJ;};YtB50@Oj*1ie>Heha9M@c=Bct=c!=3&(WcK3trIf;+*BAu{g83H0j}#au}}oTY>uFE9s{|iq?2`RCL(IyOl667V-E= zp;8`EYx4ES>$pxV*T1YN74eWvLM`c2J`(8ofqB*e&oEaScmJ@K4~YkjgaeJQ^rX^$ z&lk+25pqzsi$r3%$7?Yaz0O2da|kI0EWF8VR=;d$+7VJ{Ak@e6Tw1P9qdRzEKRSHm z`GX~idVTk|EtcN3u&-tjSsYR9c-JL;y}Acfdm*U7L*mzwFB^i8Whws!<@3+X;c#63 zGy4*d)_krRHGTQ#WfwB~?536!lyH@|7~60x$g2KR2&WnJHr*IH>-`2wN_a9@__r|Q z=;%`-5)^QKv>Xg;wV(Hv^V7cU;VJq}T@%7>zM$9+|otIGVz z7!fNkH5UEz=BF3(_~I`@p@zQ1UTr^X4n}pk0%^p#!#0QUqS)B_@SgtFK+@cQp3n3q zb{DhPUE(LUJrJ9RG$Lyho7A|n-q-7&`hYv%^=YS-(lWZfp!pz--~1l`aR8gajC0oE zkelMx4~k?8T-Od!RxdT|7^BBnOoV9exgn%~NV)c|)LipKphZptm#I86O1__NTPgdU z74hHUqia8N37aM5vQgLsZGseQ6D~d0whtyaeLvh$3%+=K`qd+PwiWYF23p(n`{VjG zZ^h)KDjvrdAuS^r{b}fLK(<}F{>kg}cvDbpDD9R%vNJ zvixC}R5o3sV<5J1)#~nCmq%ZxtUCqMlBc|dJ*WN*p9{{tNehJ&0SEN&pj|p3=tP@+ z0|cFu!vxVflQq-qGUS@1URepa`8U(8V3KShPg~$#FE7b-OV85%tt-Du8t?6Lxr4U8 zr`b^ui5jwdg0b@)FDMU7^WRticdQnr9R&r3@T{Au2Riz8qJuY5rwO!iSd3*UAk(SA3$MQ^rM$gj$1V+#S|& zLMTX=YcDwMT@wl41k=}O+Zfo;i(Uye==ku3Nceg#??%vH9dgWTHX>iVl~9yutYcNy z5jV$)s_(t$x|cy}y5+Q`Jf+_Yjr?VV5q?c&9$W_6``}f3arnutc6l=13_>ry*e~2GaG0?pJE5I=M@+8``KBA#FJvJtSjh2rmq z`ciLx%%nM;Oq@#iL6;qXs`qS4T-jDswj#iGj&gR|sJ{os9=*!I#^O3^iN>DA!?XH=_?1cE^&%*P)-@8H;>~AJo%XVVyBw z#YjA*H9T~jba=y)pFVl|1apXB6S4_m`-qbPmqO3}*muWtO(uAIQGM=gOP)Hqs2yx3 ziTBekALdF=m~8x>8*1xSxjvg}XK!oq(So+N^7Vnsf;I8srvh@|@)@&CwM=b^PY=JP zxL%yuY;A6pCNE_cln|s_@%WXWtAo0Hr(y*E6Nop-%C4>{{Z6GF8E(%t#_y(d0JM{O zLl`3ioI~n-uPDhvsFTXX&BdPk#$DR1)o^*q2E%C(((biZOpwOB=r-Z^j_(}>Uke!; zG)p4a&$8yec^rK3y_dg>NWTf8e%7IpzC7~*=+6RM?!XNq!w<$Q%(_$ajaFurJ`=^$ zLRorMwSs0l9QT{~31rVHo88wK+NFA(c_#BVsu;r%FJr}84(hxtNMta`(?E)P$>r>8 zcZ*uMc-Fy6Fl;K*?U|RFN0IJ}(1+_|Med8dbNPwykgH{p0LDmDd&ls3C#yYqfFZ}% z(CSVuscvs$DI~LaJyEdXn8r`gK{1fhCgIrXtSO7j_F9s2<7&p;r_NO>Syl;6@+J5! z2H%81(VXi)Crnq6^=BF&j|44SQgV!?V76Yxh|z)6;9GmAcZ`TTEpTzZha32=&HcDX z3{gvJAIOOCk65y3#F%|K`_d|*@<4mJ6cfs;8C*yR!r*L;pZcjMx9WQ-r=fajM%=F5 z?|)z-GVRAU^6P0x%aVdBs)xbXx6Qh40dC*gUSck`Ii?158WB3e*p|o+@eH*8peGc^ z%*D*1{tcCVEalY+*l$cc4+_nYj1QdzDQP^eVkDD=ZE#+JO26bBpf4J+ym<_cyI}t~ zLm&%E4?B4L5mx${h0?#6^ef{dSyI*a8gQ7sE*_pC>aR*{VpMvaW5RuypkZfOUUA2? zjv&s9^t4>gt$>MEh!+N?5zC@I;1aQrAHi2NCms_k5Q(cz`H}Pkdx&gB ziI!`_vS9=k$mJ|sJPe~o&az|+?y@V2eSexTMj>3A88zGyAS2LW6FDo{+NHN57%egI1t&DNK@hG(If^Kw=JVzzDIh!HeyVjRALxV0T{?T?Lle*_@H z8lyrR@}hi%_K70zr?eheWmSI>ykzku*1lZz5hN_!lTvI{3^Z|JQxpsvv52i1iczsr zm*);;FV|98_3eQtmns9Og+M5h`{QLOCMJ;7g1zfcFP{P*SD_#hWxy@0QUyW^d{?sn z&A~%)Ie+(K+JnE*`=co5<&2Qft|=6L)d8`JHT&R+pfY(svB%ga!S$ZEIV$dRPI7y# zGQLiOEPs6&P!`3I=cDHG3lmKZO$@C8>v|P@t+|WG)RR`^Sx;EZbtwBg$>FV(4kJx$ zH@&TaeNXcnF4v>O5W4lo)MI=5nAz7MaF9yNxWm^z`=qkJHE+|goFeQ~Rf|LvOAj-7 zLb4yS!bAX#xoR&au+9yEM*uEP{hLE5e-<5zc3~r1X6JL=#MALHxN$LH`|P8l{%=6^ z#qiegtTwT26gKaqzggdGT2($Eoff#DHgAzODz0Il{p-l~;#2;=Fjwx)x7P`{LB!2; z6c8p=kxUoqP+hI_Vp^4<7Xr@&XfDt`qxCt!KSRZX+v%PvmzxuGI9-1uO!ZTE6eZz( zL!hy%>FhO%u&dWz+3XyqS8F9(!jo)~p-tZF!!e42b8VIx<6(Gr!{dE$JDh~*vgKN; zEVN&!ze2M#S0Kr~n{deQ(@D~3sWCOM#D3#Ar{YVfIQ_ZbKn|MH?zEIo?T261{(Xvm z@;K%R=7dq3*;d!Jz$$ntKfsXc3TPP#=h-talX87TOqF=XeS*B|s;l9a{lcx5!CjKq z8ZTwDv4KK=`zk_BVyr<7$YU&Qh#~C#xQ5S5J=juf_87j1 zgA%$Ad@#F)xl$0yOry?i(!I+A;-JL~D}$=Ff_XC!S_7L=%bmrX_f_1hG6z1T$pQ1{ z+`RQX8i*N1NW$Rj-2vp5Yg~_M`(A3RjghB<+3$7E)Fcez@9{PKvVZ+Ejs{w^*ftmf z6&YYerAZh>)|ujASNLi(%|K_z^enRn7iD_);uIyWjr4gHU(O_9X1CtPW4oN^d<=CU z(fLC!-#22|=7W9J(}@di-2h*bl^9E~5f%SZuLNg4O%+s-Z#0b57l_XQGPs2$`)IDg}3Z3kzU(AzHhAu87r@vc1G{v00Z0A^Ll>P z;s3dnY@Swz(`w>?1i9tsoXo?3nH&qU-k=I9w8dLa&37(HMENw=8CipxR~`@?OZx1- zAl%r(-sECmSBV|vEwm>e<>tWo1`W0?6qOIBJD!h(guqd&-bTTL-{z%PmSj#`{{g3T zGilsiC7ZbyzFu%>nws_CRAC7UTTJ>0jc47q({Y5kM_;B3rmr|^mF}(u7C0WT)V9J$ zke|bnpXDA2DEu!lc26C?)cuve47IC8h5KUgm50pq?tXGVX4AH8^X6sKq2LC+`Z?3E8|jJO&&A!#yni1fM`@}BZ5er$P3++W`RF)rUPX)8)(CKY|R6x!qEu=wed{O~nd zI&};O+{{%D*yvl8|MBAhDMLt`kX%G`AfqLtxrX19}S3X%i_}429 zU$qw6AIUXsK0me%#v!_+pI*a49)-cWy?Sv;>I2kPsf^yB4>LGQ+Goc8DQ0Fivd|Re z?JnNdxx_*0vuK%#fsR;m6)h$6a&L-Mrmg=wQE%2W<8YT)%HwpI&=Ex*90zgsmu~!6rg~x(ua~%YzSC?!+HA+O z2kTY*G%O`$raN{a@S+ipboH9A0fiX920Nh<9w!;&Q$T)4qqhV6yF4XJ;yS@rVL<^7 z5l*oG#yRcOJT>Yq)p+XqIEjU~ZOz0YKep!|ZLh4_$xF8Ik8~8@%0T%jv+%H5 zgYd*EQOafxr<74vJ==&^qr&dEj~b|q+T-I-I3&L+xI4oR#u4S9&Qg|CJg%jreZc&G zTDJ%+afTb!76$LvC%0321o;TWS`PgL1;2DNb-(tx&N(xW3SkniCd|GZl0>j2gxJRM z48@gh#gZ*;(I3-0y4g?1el5OI4HCB$c5&ra>Khy2C2bKy z9T#|!yD<&|*2GyK-dlguLBFE(@bm`T{oLdlwrB2~|GK}j&sB$HOaJm|kpA+sxc|?f zz`vK?24ishEGFo~irIK6jKi@2|&XCUYAn*uh~KP4UgI#K>ZIif2X(c9#uh z#D2$s6HWm!`V5l@p|twM{qwj#^i3W9-)-8_Iw7p9h=Zwx^G9wKf2dZANmg9GgmjR% zCaNeO}hpP zel_ACw#bFct91F(kN&q^itZLp`v3POj=4hN$>@*JKijuHqTMdyubIL>g16PQ*Z6;i z9NBcKL?XV|YrtbLu zdYMhX3)!7GlN}R%EM@rN&MC=xTT52<@s2jFu0RQGG?8ckC1FCTwU2+7gvT`2m82r_ zIQGg2w*Q&&6y?u_vKZpscY8i`IvxMdZOSi$9D7RMs_im(G(Vf|I~*J+JcIbqG?m8e z))4ASBAC$RlDT8lEiu_C0kC-%e|oS#NUWYyIDzj<(-o7Ls{d>XeQ>M3X2?vD+ygqh zx~UJ?6EUI`$F8Bqjj)$uVedQbzK7LrwJD*N6EDsVxrWzGpl7> zwnz_L^r)Qq@CZT>Pa_AaAM`G`CjYTI?Uiu-5^G(cAWDl4;}UfE)Yk9{j1kc{$uZGG z-AVo17@7?fU7+RAY3hyRL8h4uPL#|I%ukpCNRU?20aezoPro?J{w*|xcF)sgA{}zr z^-wuC-*3KmV^K)?Dh$eN2s$U&$IG6)3_d0bSm|W$Z=$~$!@^Fw`e2vlhqdT38+UC^ z#n2{EE_y03xV}lZoffc68O#@HDG4Ccop4kFop0iVyl2S#S9OT*m_jsoM#n}5V|_-8 zkKPpLSnLFJ3Z{o+$;zAx_Tcd28LEA+G%`@f{^N9)aVH!tzC*hXH2lL^8dP z9MrHES>5sKX`@g>!O94J2<>g|Ro~>Gt=Dd!&%XaUmCG~gW=X^gmoSBNp@`C~P^bB6AX@=S={s%SJKLLaJPL7AyY`$$gTBsEHzRRrm>Z82mnDFQNk|~bq zngdhsamCJsI+kvfSfsn8OHxXZ5NQyB zl|~vQB&0i(#-;ndet-AgoqyqZVBa{OGiPQ__I@fpp2y!sVoO|Id-MWoN8ILfc5f#O z*}bai59PvzGLzW+sDh5_{uFjFeO~zrM*Psc;I}?VnJDAtWQ7;c zdOi%*w+vDb>s6yUBaFrxuc=td0s=7h7`HHgyX_~VJJFhdMX7%=g!sY>e{Ony|vr!qE ztQvt@)iRn;w4d#rA zBl&D$RvS9qzl)YqtT^tw)CXzJmRh!Sje;196@KCP_PmF0A9)O}(?Q<7iv(>ybJx9g zt=LoDmLU}a|GzW+ar{dddnLAx>E6@xuxK9DH{Q8*EG|Xt8FI7EafPZ-v=*#r98agLK9#RT(PND&@!&Hv8(hv8i* z8Bg{3t2m5cl~v+h9luo`Kz^O1A|T3-=TZcgKGl=8AX4lQ(CcY2J@hRyfG`PW-W)a{ zHl$0Z>jRVT0~!i|qiqjLx~0X;nyo6TDhicb**5DMn=TE?MNrU)|z zs^}v{!7(=|I?hpV-@cVqIbPcRHYI!7x`RKeOgX)9+!OljJ<@fe;-Pt6GfOWz;h3@7 zwS}df3XL0kfzlI)HcONXZiMRbnUW(ae5ayh1}3zA%PI5+Wz>`_ou_BBD?Ck0CVF0F zph+PwV)@Vp_bCHtfNg_b?|IYRq&Or}wASF608ldegxnzFz<8&x%V!ZR>Nhj6Xu7kvPE6l<3KaeAo@fqDVR3ryn zf3*((UU3LPeI*CWpDYEv9{h7o0%`PyyH()Tn^BdoJ%5@TcZ~)n72$5hCxY__vqPQa z=O^;auEs!htp-CA@s-+N(F(ta`&5Lsacov(GSv`pZa61?DR%6tZOLxQo~ybVawr{F zb@vSdV=3pUFQu%H>oSx^u22O4c0XMvTWOVY5Wp??ks0>0uZ?^P&7`H#apF4}l2gj;){}ed?M_gZL(+c*tXa9N zWcXw$YIcjd?$Z`>3JCE2(7gG2{8iZb(%KWfPA9P;dolHg-d4#8RJj$8q|cqyB~xF4 zZ{JNt$ge*GZ_={O358U9xb|dA=WkW`Vv1Bd@4>Fm$rxinvxy<*_*9eIv?!j3jd)x% zG}b@P5DKVYUIV4OCF-!CSD$XVIdqO_<`oU3@x#``NB+o{FsWOc1y|Jc1kNgKlnL6b zRFLbMOQ0X$hlmP4YE3!qMZ9FdOfH>7*lKNdArWI*0z2w^MO z{zaH}W%9C;(2o4Irq%NfOx0<9bWTVuk>O}YNc0gO>GaJ->5r3LA@NUYXmi??hJujB zn*9}ds_$$y$c%Ry0*AuF4$Yb5sOjo#5(FoU@9uA)$Es$5*hoh~ z`Y?TU^9*=tp8m)o8N88zlLJKooV5GtPe;FfpF+X8^vAWy?!D9*=7&FUyFHz5{!+n( zRIE?(zoLrp$3r4Ey$N%-2*PT>c`Fst82;Q|bQZ-m{0_Ca>F++<0~(8|m(Pd)zhL6u z(KR*sg(R7MjpQVkN+i9`BVKWJNeDezx+6jNsYQo88I-w!LffL{@@Tn}+ZGWf{Pm zDBId=VNX!ok_CPa^CPkumEMutIoR-KCSmlP86}=iD}0?S$^$F3uT^!a#tBu@;Vh?Q zqp&K%`zKo|tdaZtf9WD0NCd+*F}=9rYGMa2yS{f@eKC6a)$fbZAd+XMzy8wT#a-r7 zrQl52VhbBNm2K#BdbF zVK>PcVR0+hd`;~LV=W+;9529ZIHJF2m!u2!_dG{_yN7g881@C%9RsR7Dbm2>xg~FZ zIup7Y?}=qhLDo4&CupF6Xyj8>>KO_#WjE z(FuJr`-KCvUS0DRw52al#c|ctG^7HmY+UdL6;0RSFz)r-$HlzlYv-!bq>LQH;Z%V2 zy~>*()r)I4Froi)jh*}OQm9{ix4SgM>+?BfVJL9|sN%g5j6L4ia+Ied>W4-jo!Y1F zz3BvR<5z{(N6zz2goSFEQ~7({+AFa)8&V+plz0=4&qYYq|4-lP@Uc#Xv{;Rc`E8yg z_M|+a^nP}Lm7efpxuBJ>g!T_*=;rgwc08lqW(X_rgQ9+VP8O&%gsA8f{v>bvqDi4b;>D$( z-tLa&9|zNb91GKv1Bc7{o|h>hL}?1T0<>-!kMlmGT_aS6V^N3xHg7J*69aM*Ja4cq zV|9PO3f-zl?D>Tvmzfm&h8j8(?v8i9G1XMzF;#3OesKn^uflFAA;;=}*+PkhJ7z%@ zW)+^8&}T@#(AZgW{p*6i74(M9cybPXyrp{D#^HuDP3030dpjYv$SW zfTHIjcl$E zUPgsZFRN;;A7PTzRF3i?u z+`O3Z4ed8w)LSGKj2Da-ikH28*Y`SpJ~ZQiqZJsK(+P44azbq?HKsm=eN~{Tz(+sK zT_9T^gFGNU=q318I?*`Ogky|CN1=sLThGMq>pKgt&9Yi~LRvl;S_^f4@1hD`zCZr{ zRiV3pP(5t1U6*#UQcl%%Z!CQr8e&vC104742$blO!`n=_(hR}WBwEIa8^bT1KV3iZ z_(Uc>_B?bRS$|KQY<1Z%H}Wnz%+0Ug<#X_aN5ZJ!`z;5@v6|IG_aFNda|=vv$H!Tr zZuQ?aiP8SYvAvrvlZxr4Wh5OtALmi8M*E3LzqMS2c`xo)eL;isq33`_EH5%OYTIg? zFqD!REIWf10*22>0;8UMx49cvAz=qMqncY(BAU$0IPkNq_NL#U3vad8cu=0ce@&DH zFwrn+qK7O!nrMZ#{q1uBDNdhA;c9OOm!g5Ja^2Wq_c_Yvf>(<%mE#!+m1%K^meK<@ z`UxcwRS}$Avf@C^Yud>8Ec)ah1sLBfG9UEFan5WwUDU4#hRtnZRI2Dt<0P^Iu`8$e z7aRblr`Axyjt1YY+^z`1&VV=s?_<&S`({iamnXWV`#WqOIM-4Oi@N{51IbyJ!d7XJ3qE!Vb zF^*m>I4{T{%ljzO!DxPSh^s4#ul+kp`8LKSBZ#VuAq3P#-vR3S`Qms@p}a5^ zKJ$Xu@U$ifC*{(-o}mlNbd?g+dYcifzv96Q%Ba`Vis(#4w3NqGUXsG?0?%{=E#r=! zE-TGdA+{P;9Hc~Vj3rF(3BG(t!$S`g{jLBI^D+ixcn;#WHS9;){OAS)4qt~%@JE&O z;_KT{-1ZVTRBvLWLB8(bx|Ex>+7DP992Nh_7DUVb_g-8Q#)pVMN3 zDiK@5c0BUeV86;XvdCP~r(@9Rmxp+VcwHErRd{dOlz{SN+H4!8n8rSPht?@FMed9%EBUz)-rA7asQrN_diWzm zPJ!?zc&x;+)u}aXu&NLqz@NXXZ9e60j?4Lxm|C=6xfR+9>OI8qrh{+OUl!hFA)7~5 zqddlubx7W|C?>>R$;G5K<%xL%^pTlYpMZe=J$kbjJL7IKBeS%Z*YIs&LD~4Pag2c= z0!&)M&^Gfx)>@GW#BkYrxe2{@)J`n&V@GoDYQyH61_%^Mdt5bMD{k3A5^Oc)&l@r~ zT{^;sBHD0K;~}7z)^k< z*}IM~6RyIu@J7X%XR~Qd(*gZyW0J-{piuS8|mAii--}_N;FP+*Jbh>(8U3t zQ#k~f#xY}NV`i@dqdD8s0sTW@*O+;z^amaa+I}$eY>pak6v&_ij)B#`VmmsI&a28!);4nM+w<-3d}jFvqWgaNYkQGD z&;7Xnz9RVW=D=R;MFhHooPVnK zuZbT62=8>JnkIIMIE)`ZgDP%GbIF{fbAyB99d2&;P%cSKr=6>6WUc?az*?0(Z1|l4!LP-ShT5BV2Kt5e z_A`P?&}Y0*LDa}tL%%1(y`|Flj-W|P4Alt68TLkv3KCKUir6AHN6XnY4{*SQd6TF$ zoGNCkjn{!9E^G5hjlJqI6ISa{%I2!x z<+w~8PDo}Y6ZxdH3ckiIn(A$O(8~qPJ%ieSQdh5uvtMY={@|f!X|I1F>}&xUj=~-u zlMz~MQi_L6pV&4^OlI3tlkjo&th*GD2p|L#x)eeA3?KEHwO;K3J5_>N zL*)p~TJ$*HO54YFIm$~Z{8Q$qrY2$c{QZ8n24gm*PqZcQKu99^8tK2WvoX%L`#sWRr`#+X!LUG}tc-ch4UB`Lv_YDqC8z)XI2(DD_8$}>wfhagB8 zb8h|k3Kt>4CBzfNS`wH>ovVWj>;=3%Uk%;#k6xt<5wR&R1YA|x$n<{bSn}JX@!-h$ z#^l4;x~OF#8-^GCC;_B!uO6gsE;H}9g3E?32O?zHDdG8S1gRotRQOdUsb=_@klVy< z=Pcpl?+sOqmHy~VsL!u@Y9^O2`8uZZdb<&kxlI$DgReeXkeuXMON_3g4w`Qy_(p#8 zR5Uo2d|=s71u9*8EmMuTXTa;Na#Oq*{9FGZLgnhLR=0`?Z^({#a?{NVs zg;c%q-`H{T0`9dhLmUen)NH`L|6?-GY|4Crbu{e2SONFLc+|{Ws<*~oAHQieZZ=VT z)H2PQA}9_EnDkviA6D7=1n_ox|N1keA)bP_q~3>TJVFnT5fA2W@&SW&`O|c-z8d|y zyMK#tw!0p@k-F+NUqy`TQU+J)7ibKe4PAb<14QFxPQ$21j5QQ9rBFKm-Im z7mZv#EMUMuX%`p4rUcW+($~|+LVb89@YqP-R`G9stYMxhBA=j>3GKCywP5iMo8!2o zpjecjuC)Pjx%2${|#V0@p`q&$I1is&w3@XbS=mE|q*Ti@eg|NZ+H+v5AH%u5&5 zim=D6PpS5eu_hC8h4LananfY89d8nR)Io5)v9#oH!MK;H?0x!-r5j?7PfD#dh$NlNeEA>Q ze_V)rLyDdpNK@X*Z=tRc^NVQ*HEi?dCR-s@0W!`3%lI&`KA2j$!ErPoB!)Asn!{0$ zZWP`NenGYQLQ3@VWs|8h{|guY_dyLPXUBVuOSNEMo~GP=cl526jW%DjbV{`H;Z-@< zJ>ciqjXDf@T4Qkb1+C;)`_e3%{DGb#2X8rozvCUa4PD#w6BjKcaJ`Wc^ereZ8I*80 zcj*Ia{hd)m-H}|>$Wg;_lz7P5J3|Xh1n!yP`jTsA%SOvauMM~iRncdGK`}a>JuhI6 z(iRS^EJ2wR_3z2|S2!#P=>EBd>GDPek+XK-bSu+KgkT85Yiu872`V+bG)2nb6_xm22->&Lm`x+!UEP`)CiNkc( zlHPda6empP#uKb1wz8AZN>=TJ9?1P`0^fSP-=xn(UPn7M&yzv?jG$tQk?z1> z@4hei328OYih-oI*h=c|W@gSrGGSMlO`sa$i|0Vrn1P=Q=FfKg>#}C;?zKozAeRnqv*0v8I=%r>zNtj%eN7iPaqZGwTDD$ifHjl4hd}& zw(vbIr9Uqx6kQ7!Am!mOGC85!IuBadbP}mEHgS4Y%Iz-p2l>hn2DynI3E&Yy!C@=x z!h_Q!Hy3 zTtiCB7KMprMM<~F4g!R+kQb6tC!R8M7uEc;-UP7Ef$LMawP8k_!g1#zHvzf>W;IpF z`9xPwaDN=!npW~n;Lg<#Vvp!;{1F-q@AKC*i$alVpkoL%n31wg=5 zSQ|p?JWK1w6Ae7$;_@rqOA+5NyQS-#EoY#Jx~uTRx5v;Wip#zr0dwp``1l6XZD?~0 z6FE*h?iuTYMbTSOvhCAVKpF_A*Qy^3{Ek1i5UJZrcihtyUMJ@W~A3q z*ZU+1eOprN7K16IMMlJh&vIO6DrYFT6MbxUYj!692TR!R^srnUY9Xa3>wfl`%L z;)MDfm@kI|`c|tU;O*Bg2dg&h&9K^*3M8=*bm*+0!H&Vl%%C@vX;A`R(1NvyR6xXV zz}1T4empT{44S#s_W)BGv>Sh6s<8%+`>%hn)Zh`)d3l0Um`dO69UeZGA)En0+KSNG z5)j=Ghbg*uvwvgLU|p}3u0@K#Yfx=g#l-Q8_W1#+8BX8Ne|si$EH_q{+F&=b_LG&l zAbm6~L4z0Xnyy_KpRsKnrZjgqO?X!w8l}jj%cN(4!{1>E^dH7Q z?||ZsI$~hw*ZOpXk;V(A%OBgz2O>CV|JRYr`D08sE!#ldIH#xa>AfLU?zRv zH55Y{^BI^K`ZeYUI$v*)zrtC?uS#k?o;vnAc90)1k#4$UcLjdIiL{s!-Qx`(3@@6~ z!u$b=%+fylTxV(}^2cVj?5h^L-h#PPK9sEv~#4A0RCjw+s3dQ>j;;Gfn6b%yYLl zk7abVqcXCL&*(9G-|j?%5^(W_tfIO3wcV+>y7?A2>LFuE!|A&l%!g++9n2X5?GAe6)F8Y(sQ@(gt3~n z9OabQ#-CJfZ@N2-H8d9(MFb}9Yybmh^CyPCN#W;xxbV^mHVZ?vx{ohwr9-E0GH;*R zVFAi<)U>$DR0=9X0Tfl#;U@3_Ey*e+WPPni38bsosD@UzXxl&+D))`ivLk`BRD|N! zBu;42rJ6U$R)5Iw1th5gar1n`8J8Mr!lyuND3CJ2^l~~Rt>#?~oiHc))xqC+e4d`F z$*Re)vf-fO_Sg`MM5<2)kpgsID#(LO_|bp;YP%L~6^iMzmd5=UB#WeIKoy<{3CEP^QIQVyjJ zDZJ0SxgS`TYCSrU_rpXRbEAB{#FF2o76Kx{4!yt)vz(T*=5vg?0M804YS*p3-mE;i7^!2_uokc`j<8MZFw4rb>E}HH#e7T zj=wit-F4kTvC1hnh&Di)onntWcQm|ucy?^OiT3tRs4fY&+`HT>sBmQ?9JQFCxS_ag zY%Q}>vw{}6%cSKfpM9*FEFiY@5b8-%`RN5`Kwe-@fvE!Mcm&d$J_Z48Or;zV}w0%aM}`t!B;6bj3$oStcfv*zL)7Bog3 ztMVu?)Q1b43q(I_eRSIK#wszH*i7m9*7B;b^5ARp3Ui99PPP*y(>=n1H>iK6H4Z`D zqw<{*+4}nTw6OcP)8dt8;U1cTy|HN=l#$E@MH4Ne^Y>i#oU=c%1T=bWQ&>*L^JQ1K zq)8LkVUGtw>}G2}maA@tenP~a42|DSRZv+l*9ZfHKzuT|*8*bTs9awfV()X{z6z<4 z#Nf0T>={yzAIVK5TrcbpVEV>e|<~T^Gx}|n><$)Q)y=}hU`;a z$|$J@_2H_mf`AnQG9U>2kBvOa8z6V$UIq3Iki%xOm4?sv7aIp8ZI=n#)MZOg*fjrk zSmXcEQh5$7)NbQk5sE-?1ToeS@DuVsC%@#n#`qPORnIFyu-8=AZdV*J`K(@cOV%%i zlARnSWz=fB#gzbV$A#Uz#umR;)8yWJ%4YQKS#iKGYTvJ&wrcaPUQC|=PdFgyd8GRF zJhqan$9qy+0d~z%8g8*D*}PexS=zrs=hyh>-?`Q-dB%0{uyuF086^)eln=Q_UjpM4awy;lvEg)gV1ET!%ff{K@qJlqAZt-yajpk3d!;sjr1VHOOkJWFJER3GjF+01xSpf3$@5RsESa%XLVV&o)1Yuo zV|nq*2AeqEs@dmt{^0Yl`K>y7+U2U1$~&wxt;&{JRe<9&=s8G(wlFfQ%*tX_q~UMn zif%fWeALsPw&%2v+tRVqmmTdn$UV1$n$?QAMRpx-k7r5^y}d2Q&Dw59N4!znWxK{P zM!J!^>N+Y?q^y$MM+)WDKDQcKQ{1rSVF_5M!d%FB6<)KgMNRFc4CZ^+JMxl)UP;&SCu*zp>%|s&S2905r0q{@sT(h zB)G?J zzCWR(6v*Pj{;f8cz;UFH#IKUh{j2!P!?(C?!PS%s^+q&(0T=Q2%|r5AaZ3hMKh%W6 zcfg-eVS>Pq5;a7GyJzU?7p;@=d*8N`U)(63Cam>wlRf z2lHRLN8F}tyLqPZ7LrvcF6-5GE|ql&TLoVS)N2zPZrgK|^2byXUO{g_Ti4c^k_<(;ip_DZ zI8Gipc`Wkor%9s`xe>bIJl#VCTQk-*Z8y);EP!C-55VrGm@qs`P_J2{Mx0vsw3O4- zM|4#aXYi|~8} z6b*`$mx67zw`fHB3>U94l*?Zg{J>=EWGhO~k`&n0Ur$cp1gLu0FCs|beB(RNx3NBb z6h-9@K9GKJ?HLhNn)wa|@#7dMJb*E{8k6mV*#fQQK&+@l=*Fg9K1!6xt4!<^>YHb@ zUGJY?5Rw^k1FUB;DZEtIn-rh&6HRRZL+<>C#<7`Tg4~-G$%{hf!7>FiK6SCD8rN#) zupz7_dxkfRr*iOu)cLv_YsHZ&vWYwYM!F`Y!g0CT76MGjK(%dvF-u(xDQf804iDKD z7Vi4k*CNevtuEyCt`>hQ^i-?jqS*!C>_)82Fc)QCQ8LG;S^OkkTuk@+w)iDs7!YzI z20Tg20$k`*zPxWex;8>&9&Tsc{^Pb+=5f8n%0J(k>sEWc>$^B_73EREsB z1rDh8QRZqdDABaPZ$=JaC$GmLm};=FJaGV^j{l@DmQs8yXHBdZYc2N^(|w)sRn-JN zHr|!1r5>whEGC~=0(^IQeSEr#>DoDaw|wZ6q7^^2A$XGDFkIc{x8bU0x`4{<#NHSg z8H9biaVh(~47-3O5GM>5Opd{YHtSz=ZeEFrq_0;xK>2XmCKpqw%oH1mkbwJa96=|4 zmK3Wf$+~@Po-hqPH7KhdGnHLoX^M5RCx!MSZYQ%kAiyFa9-=oQJFY*>ls(lw)IB2$ zYh=wLTEh1+vhZtYpyq7k!?>KG2!0^4l{|NQl*o*`#l%xcc|;vR&wzF2GB|CYYroftqt<4y*A6 zY1KPOmQf03ltkskrZhCwAsaM!O??hvk$A10v1YVlIPZtpo1o`@nGr5g-CNl!I)&Vx z65D~T9e9%hH!7#H2kT)70!MYvvqV%!(sL(5rcf!aw}WK!rR6gajti^1QI3hux3f&0 z!E4vW%Q(?WkY+N*4dSTDY!3lPO`D~u6K1rnA?wc*-MgyBYH1AeuX}A)TUUER-yR{ZI3 z^YQlU8_Fkh8Wtls=+{bVs%R2Mhx4W3 z((pCmpEn&|EQ4=<5Kn~-I|5^ZLl_^l-ZZnwZ`jSQ$oT+P!TRw z-|BZ1Q=S6GIBZg#7mPcvJX~S=+FGy5g`Ux>h&~CS-MC~vP@BUKYd~cJGj%sk2+ayn zY%%mDBXtTR`AfeJNXmUT;_wZ(p^ZSKBdQQw$`#pq0_7}!z~^B%P_t8T<{wi{AQ@@CtM3<^uM|0+nm zRc;+8NmgW5ETPaU=dW#|Z9;?=;B!WBfx(KqEXs#B{Im<+8IRf3p!|JRjA!FvX)K=bqfDD@;H!- zB=>gc{g9XdI@0+b^f`D1DW7Z1D>f&=dcqgkwi$?cFt=p6FSQ8UDLEk$eW5EY^bYhS z1W*kPB>#vWY&#Gyms-mW~e}8+d2xOE3Az};oJ-pJ`Q9+zQa6{LZ0+jEB z7P%NHrUroG+G?E!*V3YWQUMhA!W8uS*=Y@jBR4lDo4NJ^wWX3t9w+oKHIRK~B!FEw zOuRo2S|X3CG0YfkqAn8an;~M}+*4!TtO)y3{P~gDEQJRvmyE8xIlv9`cR@9c>1j{Tp>)Q`j2rG81LTIl;HCF<>A(z*0&KYfyWy^v_V#C@~@mlae>E8IH$4Uml|8yLU{mS-y|gEHs78W zm7(9(JBO&J4<7w922Pb<$)1QT&!r=t%RIq)-FTZMn!GT^rI3C-_ z-l`1_ApI5kf>?pHm+T*)lRzY^N;EV?7XV3~h`5XA3KkLt4yFAJ+88$C_?%wX=VybiUI_eb(#U7qdgW|deer9zfZvGjdzr8auY=HgvN@5ayVg7U`j5rxs~QWR z&yC~W8S?al)w>+@4~!k^HR2=b``vnhyEsT{K25>MDQV;VG&>-m&St#TqAPT@eNT8c zVsVb6dj?mQps;oDKDo01$V{ZJjHz^usxD{;lrKvk_W|NJxV;Aa)oHCp(3K(}Lq-^Q z2o~rKmDNyGTe3t@u^vwr*g}`yZz!E8l^3?#l%93QaSAxV7+p}<%=H#9ER}3w_@NsY zQDd?_(Sm7!M*gMPSbI9+F)a<%oU!I6i<8PQ128-Q)#a$5YQJh9s4hX$f2IGfA*aPh z#Brf`6g?o*wAuwu;0FV;!P$gyO6gp0|0kr7J#CyZSa`q+BLBAJenndnHTgOUY`3(Q zsnRL?+yW(`{Tl%<;SntxHBu5IGbANgM3*plxw6Bclm*Y+h-A6B(1oor6EAuEX)Ks) z)KjJq$fS7j&lGn(+n15*WLI8Q!C#)A3b!kcseF}T#(iljQu-(D-A#P^VYy_W)BbtB zd^fUa=R+SK5s!70?;_qt1qvXxI7ksz$%S+aeRPu3MWn)+H&rpv4DhR{jmRW1{qT^K zXvk{{sTw?HpPgX@vKy1ZvRH@XYBO;;wsn2)>)d#aCit=8A?toTd@(Es)0rN0O;IoV z@K(kjmd)vE<7AHsbXw})v#%P6Iz_GE-+T_RBa`9mPrh`so_>^i$Ii-_)YFmP+I`Ja@mz@9N z*Cr3b&qj2k&kYpe4>!uQoOnUw2zt(Uz~3S;QjLyYJZ$4#_$ z0#qy^duH^G*o^Xd$+I#9leXdl=)F&L#K14KmjQ{8o#p2*6@>zeek4pG!`{mISTB;Olp#EUS+|ZcQf(=s#kLMV&-61W5QJf}Dr@XwA@y70*hEkL4RUP_WAd*M z*URHpU7F8lcQm?OUS}B09EZa&+Xqp%9{RLxOBl_QGVig-bhgNJVZZL+6JPn`DEA9w*Z(8Zhxn|r1=pgA%GL&7mSga ztlF7Ig%zw+d1L5=5)oXLU3rTkDD;ew9g}UF>u+c&vQST9TZZVLk8%4kU-fw`Cs0&+ z)qrE8Rx#qs^^8Z$3>b%%_UG|I`;*-DWRn4(4*X2~#ErOJpsE1xPn)$O&zCJ&*yjJ> zzP*!?Yv?LhXa<1Jx8v|{)4(D`Dc+G7e~WW;USo6wew^y9(h7k$Gu3p&+;8krgzUk+ z5IgHR(X!HvnlU6$zP|oD(FXN{M5&r^tkiJGy6X3gD^4OA0W16O>R5M7Yw@AayJm)F zc7<3|2;+M;+=2TUV8Y^ITTHz}aF0AZm8>)#Rx=DhEum0=I<_hn^!tu(5VKDfUWXb} z*$hPYK-r{PtPlPH@>5_);3r8qP6UANbBWnwSSQ%8bq)|a@VOQvs0T!ymKgrRf(33y z|A;VhvjD&lZ}IUY^u}c6bl_y=Oqs>)*$X5CzA=Zueg~o~hFpd$!t(pco6?JjF6Vnd)i&fdAA$*sKnMBdTms7XZdPGNqfTxxxCmXKgY zJt}y-P5SREyheh#;Nuh0EqhNLG$Q*~V?0o51@N8h3KPo&-q_jrQraCgEawZb1xew*Rt^Ni2@z@mDD|0-Op2G3jk764{N*1syD~M}O=diV%h*XX)y-x9Ja{qH$SqGfk=Q=A)$$TgU>+C-q z(Mu_dJ;vg)Tz0vuNR>$>o zdH&s(r*@cVLO9N((^z>9qNDCPC`jSI`EwXK9x9H*qZCbi#dUY@0%6hwhGpj59VU1~ z9sfW?`|pESvuW1f3WSCxerByj(a8R!u4~_qFa?Nk?UgUERA$76&;0Qh23tpg7g3>O zGTsmMr^#!2MY+HAP?73W6x%APKZl>=p_^ zOxj|)BD!KAGC6)*dqO$Yz-D+~xm=dev0r(ll;#BS*kDmWA2olN)Tb!gb(C!FphSQ+ z_NL&Kc2NDcsqj-Pq_bNt)DVl^E2+Eydy;)&3Jg>o(Qt0fZ3ZF?xULop-c{zHFJ)LngrUfg5(KpykjxAN#(9agj$5B| zaqFytYEMI_{Znr_>|%KBN&*30GM$io@E~=$1Qee8BtV)kE2xd8rN!X92qfg2*)F+C-jL8 z!GL*#7D!jeDD3nS4cE&G4c9JQHGpIjRm z1ql~RRx${#YP*CU^7x)+j_5`!K!co6k3n|Na)ADbP*53?m?8a%VM)^X%BlW2%sI1_ z^-=6fv)rUp)r_*O{8NT2x?_b}V|qg6R2*XRUncT@Aw!Dhu44~?nvk+I9Y-Aarz(Y_ z`aZ<_dtEqAOq1kZ05Pe*k|Iq%>>g1i%TAN-_nD}7R6u6RHN7SRi|-Y0@g;czNUzw4 z#-59Iu3e%}*g8#>YGld{P5;(a`y|Id#6DC2u6ELRB|rL!Ym${@ z;BVt+dCXhg@u>@(JU*%LlKy8Of9#E#A3z(VqR$7ayL zZIb)-)%Ipqa)d<=0BXt6;GBCw8syRMmGO2$G!peXs?CdTB_ztI+X4^*n=&Ku#{QRj z%u$G6fMgG+eN>e413S>}!Ki!}?KJE6MSV z{Euk9+JVD~MJ6-SxTgfrsJQUB;${~Ro9NC0==`1YD{4?!_p| zoIlm%A+h!)OgI{dClWCQQsv5EEEm}>`2hHld6>;`F35D7OEsDS%BJG!f5Kn0W8019^wxVJtM8C=gSR3D+nnYn36>+x3E2lqwA)FVQH`YJN+h&Yd+QQmP@^8QHX%81$mqw)pt^=J( z-h=Z^i!8Ccu^0CgTh$w^7Xz_KWpa2uqMqJP9xA6mU?ELU?}b$S$?9WZbJ96k&-9PG zk09eT>%O8b6nHt0Bg5>EA!U%Vq>{>DC4BF11A!=g@bY{~lUJl;^yjlFhguzws`T)% zN$efG%9i{CS=N|;fYy`Pag7=REx)N6zxC?WQ*kNDpPZ-cvMq+53s*Sl&gPpY`b+Jc zLu~}GKVf~MTjZ&FW5}{(6TD#~kjtWy$RYMzAqqp8Hm5-*NKk)9AGfw1R84Ox-x`4! zn>sHGFMHH>HG4bT=ugaQ#^R`e_xVLbhM93V>xw3elCw~?k- zdiwj^XWU*UIzNu0b!|{J6+@A$vtM_rFCWok#Baq6h~E1A;p=mG+r2|YQwCW#%5v*F{x4H z1M~_`qJB?i)$rgM=e)5i-RnTTXbIe;1Lr}OY&|znB9sb{h{eXo1_CqD}gUP z9ZB}5oaCG$XtePv1p!xKw6Nuzi;Bw$bJXezTojKF#y4o0`6F2(@ldiIO#Mt^@6eUf zSfp9nGYI4DQ@3u=>t`{nr_mgWszv0uUAhgaPV_fF253zm2t2o&rkkb-&Vom7@wx~vI>|qwBeTBGX!c~wfRsKm`KO~hN!7;*t!1e@Fp^M@H ze_mHwbGd>uD4FA?knC=AN%J_ZdknUO=)+am@U4s?#ub<$UrT8_Ul@}) z@KC5d$a0HWVyf&>T6ytQJP4^cjAQa~SyfXd=-gX2LfRP5vv{#7Y|NXbH4v2tyG$&U zNNGR*^3PEt!Qde5*U$g(t@WLv+qpC@CaPaN8Iyfy#Qgj!Xzdazp4iTj)sZr!7Xpdik48mIVGfDgyA4YS zw@Bras!^y-47^sz=;y@0*8AQb29-3qtt=6A>ky2th_NqW9wEC;X=Z>jesPd&(_aR` zcTe?xmYnEPS<%cH7>yrZL#`I+e~1qdt0W8@#$sv?d^;IaX{USP;4VZWn-TRC@^7#C zx7qZY`H8LW#2`CQkms~@MK|lZF8BHgVk{QVKeXHT+5HUVjSUCJXlae#(K$C{udV%R z=SXYblGLfOXfZOa#>jgn;mO_AeAvgFT~^skUE=5@AfpnMauxRtTs@BO6`|X;ZmUGk z98D>Kc5~KQag#sZ?jC)d?2Jd&8Hr}QCMxt=O!mRnvUCI7C{5>*=CL-2uwoXN zj&q~Tsw0g#)iP54`!R{81H3ZJSpU)%{QeSrfIf#lwaVRVk{C9pWBSYoQo@J(f{EuEZi?f#jf|88$MQD07Q=g?~w76M^W%_&Z1N>Tb%XBB3fW%rpsbKBRah;kk& zXA8|BxD4LAbvu-|->e@h%aJoP>(Dl%rP>rT_RdCMFI~5;CCXpsG|S>$`1cJYQ#eRR z_D1GW2Z)os>+soo*-cp^!R!6X3fo-D!vK6S%(YOT5UMDr>xcJ#VDfBepy8J& z+V?ih2QRPO=ib;U>LrOpee0qeLVXv`G}}2Lme_XKj(h*3btNk=^v%2g3ri}rRqHn? z3|PAZt3nnvyLDV%7=PzKa0@Q&SCKv2GiZF8WGLD$tWN%HtQFaCOOn4Gr)j6rlsSyLnX| zC|Etbe7|v4b`s+%@-lD@8hR{Mbvt)zkoE~2`4buwQ`21kDh4ctOVdrY-tG+fM5o8E zVY_(mL{ENlyeCHgjO_y>jrR)%r{Sx0q>=+%_Z8>gzkhf~e-U31h`(O_N1tqH?ebce z(Aj%OLG1-(Tn7W$B*+Ovn)D*ik}SG9{4i$ZA4=K{4RJLvwLdIXXMRDsntt62NMRN& zXh30~FgAm3uY0qL4)$thRi#`bXa_v|#c5Pn**0~Ay@1#_y6s5-6A6myKOjzt5;Y{H zLD!!}nG--_Ye`|(DhGkcpdx*Ss*jVhD~Hh?EDv~k=%&=ZUUXNFE03{{JWC`x0_E?X z_EGjI`Y~wAf*|kWW?8Xj|ESxeV{T(^A8!-Lh+BAe*&@k@8X0pLxc@&uTNOAzrkg+L zK)@Ju08&coingNi4|HteNKl-CZh#;_l*iq0tsY}7{B1!i!D%)7->ii*v;skiM87Ae zj2oDYJ4DE+3XIGdH~SH7*PLbkAJ%AnAyF0X9v5)`*X*K3zte1fSP$C{+iEBa#8l`| z>p`kfN|n?TvuWPn#a}5t{EXAo#i|*o@;`8BsB{;Iqc|;JU+RKeIRv}+;kl} zrs`Ae(eh)<=d+^A)HWs#V}4zmej_`y{k-~al{cXK+gub$O@Qv+r@q&+f{Ad@MEE*N z7VUC!V0FZUzwJ$yQ^BcW(iD~fCz^NcEh8%t74pOiTmCZG0C6+w(n>+HkQ z3Dx9@+*nw;uuFwa49Gs11~TpdB(|^S#+#l+alDB6;H;$rF^j5T9#TIANDjBO@ss9S z_e4N_HXmxc_%x91sirAd{M7kY`UXl)=R&-VvvS*z0~gH_=QIK`4(AKcIa*+;+H zpHL`wn*$T4E*MQ`n7j46fu^F@o)H1J(}1?{H>9G2+koI_dXmB0KKMHkmHYX)FP+Uzb+Jbd!YN;zM=9gcAtw`w|lrwkIbFsc^!y?#Cn zPCO*ghJVo{OEHI0F}zj4IZ7$*=lW*nCw4IZpSHmCRqJP8LfA#i%%Q!^(c(yO*drR! zLq*6^PMS?UCRr9)u*`6KyBL9VM{XW@v4es+jX8ZhO|ZyQVD?-FGASH~*=*<5mYmH?3K1=>j(*+x*>B{EoJ@ILXm?wJM5lw}| zxh5%5a1)sp*cyS~d#c;25<_#S_{;sTetd6TFiKsG5Mpot)Mkmm_ zZx#cbAhwS%B{t|?0dHSb@6&d{20PTG)X`OY<0cu|t4s$oU@eYA8jZKG#DyKY-;zoL z^wF{%>;4?&RBFEphtHsBwzdhPn)mL*4S}9t)`>ubhP^HX@h;Je464uK&+7I_lvP(P zVr9&w!JC5B+`vaO_~5d=v$ONI&i(x`)1m58!+3L!mwZxb%zUN5Ho0B?*^6KEppEBg zCdVvHyB6o$|15estGytSJVk!#JtV?`}esS0Q=@cVSG=z#>as87g z8cew=lE^sw>Kn6j^v9|Pqx;6t*`UT-p~J9Nr9Hv`sEwq+KzZm-_~H`o0x&a*4DG`7 zd2-Ob`2H#llZ(jk`BFbL6~wXp9og1C6)I=SG1>3%ep6CpvpiG_S$hnHBOu(e*Pw-)Wy^#RET129B+=wfb;-B2BIyw?{Cfg`A*OE51>#V&pEz^YR_~Rmbw^uqDuXE{FSZ33t9)w^^r`vRlih zml*ryCY?t6sy)tkJIq>X$}0jqv!SY#$tUW8$sIDM!e=51ZPZg_&oq3iaZpoyr*$@# zskYfCkP_$+0hnp3d@(H*ne9R*1)hx}DXMh`4p>)7C#D*_ETGnQZ}M2uqCH=7b+{J7 z4>J^(se4;=Iw9=*eA;Gb9kmVZ=-j4_11L239yJdnw2wCRGi+kI5C>+%Ga;Iz9rYB5 zh&P*QD03$7q;Po~{2iI08mgskkXmklCNsT`M=QSK9pKQWl?mu`6?;SZI=Tf>kD9kH z+R9ss%V)zwcpl%Z^#8Ck=JM;wfMcC&$ymG;CmZMfN_`MZIw|^k6;=DU=Ra?q^&rLm zkI+q`_QiB^J?^3I)M=}r1}j3&SwF6ytpqE#x9UPwP=#SlayY6&{grGSDqT2E{Vtqc z0ETnd_p0xvhrin!i{yoM_E2=h--z~LQ(}uKrFC&R87QB<mtV_sb5ULv}XSl#t+T zzP*)lJls7&JOQ7{Pr&lli?7(H={1pd=laQS@v;TkNwvf$+~Vh+F6dfj*H>Oc^d&J+ z>0T)kTZABnX%AkAQ9FG3CwCT7^ zs=X}ag-ceXIojE^VCi}1HHzrtUHB%CB)*f3^R3%#B7MD~?e`J;{K`~^)bF-=@SW$R z&Ct8pjw|dTNHJ+V^mH8x0WE_ymR8JGPM7E5r83wxS}1g-LJ>H;hd4-xj?4+UWwvM z>FPEcZwg0s*!KD+c?kYxXO+V#KvR>I9+nnHET=VL+8rW}+pjAGE}wVE6h-v<$*&nC z3%Kd1Hl$}j<%vQ|+8?-%$|FoeQ|;`&B81z+_hO zCFo}$@f!$YH)U1@sNB(&%T-)u-$wtQ9rRtC-8R6=xCcEPJ$&`s7ms);p$l1Crp8(4 zM$j5T1hAm(+r`jf{d28eA=cHsm1Y^xYlfS0@7uNlaolfLmh71OnkK1uN+u_gpycwF0%Of zf3xE2oq${MK}EpnmrQv1mjtYhfJzT!SE$M{>Y{g9(C`YM@V6$P^yZ}%3rRf-p+4S{ zUlx+r2^F}}+FaNl!_Fbw#LCkji=aNp z@7a5lNq%b<#nVmd>bJNalz!QnQDT#sYlr1TQg1r)4)h`YAE32n2kEzu@8EXDEnvC{1bGW7WfQv^>B}|q=yPkNWRIg!Nmh$Zz{oeoNBpcv!3s?514f~`+P2y_*o1?7ZOiFt({1xtw#{|yp;G4qnAQ7 zj8EkfYgB601)~=221U)K%l0;Zc^k7K^uK4f5zQK)L=JDFl?p|!RPb~Jve9>foa8vNt$PCEw&UhbPg)oNd*FDS~^e>P-# zbN_^LLZ(n{1q<1ebEYQ&6VA(mXo@b}a3KDwd@BQ_tLAfl8h-Okkhjl)KNo_ve{V$9 zcTkRo)X;DYqL7{Xy%L2^IW^{#J8zaKU6#6QAGKz}Gb<9X@K^(6vg_HtT5GCGrhS=$ z>#kCK(4WJa!-7V}y{q?E952kK{`UG6#6xFyfN8*DdcnbM;{^ff`falvJ!5mYI7&Ly zmS-c{4XGc1N1UQRF>HnPD>XMpWh8m^@K1WvkO&RIlg3crdIav9Dw6FLGs6sHC%f=c zj#z8}(m@;%>YPXYGp-NfkMHIlEsRaH<15UvqbtfRbyoYQR@(=roI(1`ty9k4c{;uc z--USgpI~smj@TvkhMpz8K|XV(pagL!W7-UfG3(}WJn9xdtv}|8IyYa0cHex9Y!$@G(?@hl`>Xu&-S|(&weSH?8|)Y z@^d2NXe@OY>aIccGXu;{hPPS$0?T;nd1YqM@cTbg6)>HbEgxP|$W*BWRRk~U*t7TtwAjF6_p|!@`TGC3f z2^^F?=E|fG&oHq#NE-b{!0!eZJsd0<{z|UyWy=qPc-+!oEIpPDXeMqhalMI;`(U+o z>{h%d?jt+*h4S!y_OQ+aa4e3yeWw5uQ6>m0qt65$+$BgvB^g{CXWog@eC(j}U^-hQ z5@O`WdUpCM3k6gh-O47Qy{ht~o0vy45j83HWs1ZSbnwp3-b{3?67iWD9lW8CvEm?i zg;H($ps2|{Abhg(XumDV^fNVc$KVeh1ImiwiYAd>ODGyj;QgCwYpAU$Rl$S4A~M5p zRQ{YJD>n(GjYX%D)1pSbs}cqho)d2*$#*UHVnO^p`YQoQ@6YG>k;HmlFxX8H7ke8= zXL6#|@YK$NCL@Y5fZaANY{cgxzIae;?VRq{=DdxBic@>B+iLrjoi8_-s#>@Cefi3g zBuW!kWj{3tw6@aL(;$*oPm6p{j37?P6IVgQm0Td;cbqomWMnZN;o$MRonUds=aU`V z*}aq?i#bgcFJIOyG%VIwzU8%?SNES0xjllq<&E^7#hIZWD?9>d8};u+V_l%>5;!yZ z(`5AoIMgpvGy#c7Wsvb737Ds+CsUs6P!&B;9%SNHu1#1>~r4W@6H`;c1-W`U`%FYVW9qH9-q)gw8{eW&~G&7Tqx8Ac;`Tc*`7j>@jQe4GMFF-4_Ot5-CIea=Okd)fTj1JRoYTZ5I7OmU( z9cjb;++k|6z3NbYU3VFqy#udbr%9y&k}0zTA64{WVtc5$0dK{przGN9%yq@+%1U*GKRF>+`IlYUU%$}*MX+@ zSrT6Pcx>qpH;ajx1XC!_4l^21u{1Ns3^o8^r@g`N=|7}o(g}DK#9YRZNpz0`!iocAv*}#f1+pI zj33Ph^i1sOfhK3(u!HpVkf~_A%uFQ^s9wYDaQT5vf7k&_w@iFR;&`_X?F3wf1GyKS z1!~UcH~VsmnV#AYRuDWnwDY*Ul+i4r(g{Hr7>65m#dus&hkZGO2T6i0!dVJrb z4Z3+3xqyH7M4{$$B01~&vkVFfy`3Cc90Hy(bl4>ijb#?z?B&M01v@a`e9&ndCoepB z9iFddzQwwO5iQ{6r7QCPO`i@eVfL?hZsB-d!P!+b)ad0odNkIA_OZL)uFwiHPGW zmIi5Kl-jA|+*+w5S)=lzd0Q=VA)R^B6l@ic)niJV&@Ic6s5 zblI*>nh@sLR#ZB|yyUyRL3|tk(R)aXw#^W<7PKTFy}FNluYk!Gj&~S!0IF4wmDfNF z`s2d`!@Wco(rDd(-U)nC{wlQ$@g(iYS!|?=8Jqf^`kpE}$xCKHssgwRzLeN$B)>r4 z^ZDfi%HSOCn?>-a*;yD%iPzjjRd+`wi5vxMTWE;7G3{oIUqD45fuM)OCY&QOEEgfz z$Gjw5sYz{>>QjPlDgRO8wLq=wWERsQb$OJpI1!>2gf3m-ay}#*IMD2Zo{+hDjC+k2 z@zJx`AHz#%!Az{U+pKjut17W)S_UiSY@G_B%LQkWJT@rWG8;0)=8m&}uL0?t3HOV)+jj+dx<-(J2Kr_` z0$J`92|&2lv=T+r!+!}Muq5fNM;?51%H1a32*K+2;iKKTTk9pvFcjpG$l#NyD}`R{ zL_$Rpj|s6J#229%fwG|S7p~P(h(jCI*dxoz?dIuohqKu5>xTUi?JSpclzWMIOMvkM zLXj@ChcN<#2eUOCg6)uWL^s8SMaHXeh?OH^XkjjW)Tr~Dkbv9ju=ijNy(%~!4!#1e z1!#|I$+(FZ<~jRHNu&Je*UXyVm#Le-yne` zKGKZCRNDWhBY5!Rmz27Ybxzr!N9@Kv&xaMF$w!tu9xWCY#Z9%o&yTcpK70;W6#93r zYBWI8mq(=_)u$ivN5o56Rv*HwZi#Q_A2eDQHaS^bO2K0u(rXouKwG?>Y-xv-2He0j zRd|yHF?&t>5CVU7^cn0Dq-Uwu#WEtVds1~8XK$=FLg%KV>n=aXZ7 zb)S*%D`492K3HzTzK1%rPkXEcL8nPE_F|*3$&G2@3t>(tpY}t!mAtd^@$O|V`1q4P z5A=)LL_-fx*inq>{GGgHhg0tQhJJ`~@%_2!ELju%NN(cv{R z@`&mwZeKP2g()P81pAq^)^JP5*WLw|ES_L1AQ}Q(*kAYoA`2~>cm#ikJj4*T9xqI9y06=GxCdN-mbnYLH%_U-2jjO#A0F)kw3cZ>`| zyf7Lk;-__fBv|F?ufA!7-RaMoON{gq`m?&?o=K74cr8gFaQ5OrDoxMD*Uc;Puafv& zjnc0EswMd>rr8dESYUX50;b|+Ej`I2j(oq&@W;>!%(TaY|1-6K*m4{1EsjpqD1MxH z^ZVkWH>R3leJj?Ry`4?vMkkv^W36NG%>6&}`#L^ml0l2SxF7 zYGZHez#(&GOAiQnW~XH7n*H6yH4akzG3Y}ZI{c|`u1_L5%_mhkv2l~+gL<^y*o)X= znJ|WlhT&io;}Bl2Q(rvRCHAR-f`ewpQ9>#Z6RwKG`&FHQl6xQ5oHeicAL#HQkb0{8 z{JC-=w&;nf);@WDN*a3tErRK9UCiw+LdV-ZpK8asn>Y9UC{qJjfLW56c&2CrR5fOk z^Z?kL?mKY*;=?+xz*95+o!B4}VKN)0c!~7nxr%E_(kxw>`RYGXWLI1x?j*~}64BjD zt*0UG#-#WrD~-w4Z)>X50g%s>_kLKBluk?e}c^#bH3)j z>sUYBA4&YNxeO5UW8h8xmP)$;Q~8Tr+RtFwN_UK=u?l=JJG%M(h{77b&0cY2n^mZb zuFMCu?|wfjtJvRUE^XAhdElL-w8h>LqV*OfSUZHTuuCZd4{#~|{ueKbgs42Q6w9=l zLT@IeuH~3s_E%kT3(!s*7>psSGSzAfjb0QwL|bZ`u@nP(RH)VBD{9@neU8bYUZrh3 zh#QSz0|0i$VT*e-lvvdYY7L#6EpIlmt!(+tdWV<41$W{^49?vS(6+hXnJImhN%DGp zc_05FN~^KJS>El@R#WKEeu1ENUaDIP)tAoUm!TnrUZpkG=Td{TRL-#VB7u;h9Gr_z z_nbK}m+`&KWjmUFkFGbUm_Pg&x$|3JN-)Uq6V*~a9sZW{q4-i@Crd5gGnLqvP<$p# z_v*#_Wq2Q<2OweMo9vzhe}e6c>74s5fxQXyvLy}Xc~?S9hV=hZRVr;QX#c)oGJpMN z@6PU-!-PnxoTuURo?%53ug5XAG?9Jiv4N%2>T~mox!&rTN3a6dDxDjxd#gsU2Or@H ztk|(-l(;a2MyPz4oq$lk+d$TDyiJ>%pb>|(+YxT`d72ez!W|MK$z04e?xP`&%(BBM zDfs&4A(yvnf{ZmdJT!a!T%?O`c-L`zU0Tz}u_HAn5oGpj1BWBB*@03ClX6Xhdro;5 zTP!5^>Y>X%YC*KWScAhn&Rxk6`DHu*jE^WW59SiP8})^s?}eAxi4=Ua(C^J9r~5%% zv8RXUbt&**u>}%KRaa})L=8_BCQE+iKeT%t8_m~YdkJ2b{_;J8 zX@{X&RC!_>p*A(&KJKEM8zY?gh^Q#HpN0J6dm}W%O6V4sfvja$Tj$~WYuBA7qlGC4 zxjND?355xWG?SS+s44x>7hChIIuUMsg`S;kfaXQt%UWWp&yEXO6xf?L^M2B|8n&4f z#%+druH^5o2LIAn5l#M56ICi~-~8o5$~l&^CJ0;eK2Z*{%=I&~0??H*vc4z+mwxM* z&LQ&;eTYn^3jnp@)UoKn6n>-rWr0WtMnlQw0jO-G(SZNDp3hAIm-mh+`R9o`7eGyD z1n_E^asuf_mh5tc?9;6VDg*ZJfk_IjLXe6iH6C;$xvK&+qA4ySvC;2celRRJ*+!gy zW&2VI`rT{T3spJC+eRE1%Zs8oPgRnpsJ#sAy^1)-JqoDkf?>4)YbZ^?(FW!D<5e>M zRklf%v*RWMuac1&j{zvJ8h)XEVDZ&s{^?c|Xb-PA!Vy2Axc;LHNO=<jKN z@-4($0KfKxlV^te$EVM6?>6xB6<<^AI3DpBV~`&v0cE(CrT6C zJ=-|q_-TX}Lg_?82uE9+ueopAyk6~Gf1iz)H;_i3K zym&_ClKg1BAxs7@x#5VP3KeVkyK1Td_db89H5x|gIWy0T_&FibJwlvf;GTjL4qhE> zd9&M<0S#TL)aAAHJ3nHcSf0s2A9LxITbN(5zElJYJKKJG%AsnETP&_77u>~!hxjw$ zXR%G&d7b97u{H-4P8g9F5?CIAoh@ZgEJcMpqrOTs^;;L)%)-6S&MtqLtVJ@GQYyyL zegKoDN)eJ7k!eKD=N%QtH#hd3!vPFDV zN%|I!zGBXlp@tTAsWhm%22NieCc2Q0o7D%Op%any5@=#sReGZLJiF{eNz!dnO#_2tpDi>s)^P6vhmeB3W`a%T$tl{nRQI*`{$5HDgF|^qnRQWSx@D7$!Ge zhN@h8pl>ZNHhw?Ct8fJ!RpA@(%g6cXl?h%;6KB4##vas?|S0 zi&4&xNek6xu;vxAMRDhCxkACa9Nf>jz@*~o58*jw*y<22cQIyFSh$d;&?j`?d0N#R zK>atgyED0ZhznUW&8Kmao8~l2#a%|c^H4!jc$PfyoC$(Y%u3fixAF>RV+4=p%7-B# zomHMX)e$BFg)BO@819q?*^xy7r-jI>+O!$=U9HTAFLU-`t05o8@zS|7WdYp`Xw54Wl?1YrSkf|@>IORz2PiA zlfndl>EvW~=qBcm# zDr-87{883iB!9d%k5ViOW!i}<;*b%btaa8mZ7_M?Q*AZ(dmG>*Z`ZAkSfZ*)=>PZ+CW%AcB%K1=v)dXb1$ zTotIe&~sfx_bXYXx<9VMV#jua!Z`^=sd3}FR3o$w8_T$M@k2WY1CeD%SdCz{cRP{n zw!Dl%S2892M5fZu)Vl1pH7+FyDDLf05SHd=mmD14230c@cu3=7dMkT8gWyV1aY&4in+{c}|QMkx^k{SUq!|9&r`W{D)binOP z2-&%{rw)+#c1j0uPoz55p65Zn>0k)P>fy4Bj+C2B4>|B{XwW-1y10xY3wd9O<2z|8=k!@=Cw{8Kj1K-&k*&vo@exs;sNVwj1Z04U zlz2a9b`DRmZ93dP{H7jkT|&jdu!IOcO&wo$Rb*rBi;4JhSva?XY|WOaws-PqjJ39q6t{Le!oEVy%N=6NwQ(I(WMyowtaWZUVpjUYy>8MR=l&-M=tLU==StniOIkWacVAia8#QG=#McNL+daWH;bS6leJ zDhN$Nb${Js?IWI&&t`b>a75hzb|2b}a}*Up3t<neBW%2! zLdZ?1@r(o!L4=fxJoVQ%LnWJKyP$y%i~8sRoc1d!yH%l4I>02ug$3YnKhdpIH(CuS zJ`#_iA{^ppY0D3Cq26dCtGjKu`B!gWnn8uohy2SLI{l!p$^YzwB%PbZDNB-y0N2f- zH}1XK;|Ux#VgZF-?!PGXFOtRC;r)%VcF!0aFutQzqW&@bO*#xJHr_zfe802P9L5rM zH(Q-OOW-3wTKj|RaK^c7<#XI#_632!=w;=U5LvcilivuBxg&=m7KGJ8W)skh7&CM{AEjA1M;Za6Y$X1sv-Ny=I=Hlh%PPWm5nN;7;3(?)+?pWM~eS{ z5}lsCz98CvJzgj_qWypf6m6C8F_~d;ZPk6~`hA%H81vQCZJEGe>{iy?OLN7FG$VQp zt|d5H1-OhvsneOI7p)Jwn!~TH?Da%X^XYghxEX`U)`<^s2|N6yHq9tg$Yy>y*xDGU z#$z!PzNG34xa-+aan-K9h#jczwGB^e>fS zc%VYfo??j4XP;349+$@b@&onE9bE|+&VElLkT#_fjH3T$b&no^xq;1}{<#N`|E(cZ z#Iqc~XZV9poHf#F*PguPdV`po)he(O;zW8a%b%h|&2G;QoZ&IzVG2QD=!D$N4A(7g z#q&Yvz_$$Yq;=Mzy5b)z(702vc=~tYhfdW2ynl<`XY&_3$rvf3pCKOl;QqBnkm9?R zb+jmw-|t^_6w+3{sPRq7go__OmN$~$kVfWU+~K0GRO(}=loy2(+up-L`bwLN;r|3c z{|!m06mq@!ug}3r)$U1&OEoSFTlF$aSKses_|1?$fJOejI+1-b$k{|0s-@?hEaEFE zUKH*OAyZV#44dm@H2m?NV`HXFWa&2##HRKcy`7!5Y#^Ypcj7lLeZt}-*6H{lL8y+Cf3Muv$jNoIvH%g zggW8FIMG=pE!r@68}u9rp~D;pT0&ac99s3luyQ1gVI6xLmE?YX3zDP}_vXYhO{9J^y2)S$0AS$JbvG66=-^ zE%^4g{}p`wUB(-JAZ@;Fjca$ASwekL-t7{2DMGw}UwQ*M(AClOA#>&@#;l*nr^t)sJ|Nr^OApx+99C@5vIQ)x-)`~8a}XAhcag_yRDQHrIvwD z2xxz`yPlaZe2P0TH`OxsJ)}4@p_rBsrtLcL_rAd*CY@R-LH}@A z7J^=(HOlDBV^ZvT*CxIOdS>@_fyq}gx01-F$8-ju$TaZ)6)EcD+!0ntEZx|%l7)Wm zL;Z+4J;h1L^uuBr<}mdeoBrUU&s26o6ypLIlt%5tP{%*;*4i=87JxM%~ zu3Y%O|0=rQ^|Jbte45Ns=Iy;vqtWxx*$LgAHV9!rTkd5wx{CF*Vo4}-n{i0XLLtZU zIN0z_JQ?fVoOjz5vGT6R!NF&`SM+xLZ&XMD`1YXKceR76mkP*a9csC04mPod| zU?~8)!f;^(SbA9Eqg=i6a=B zEoRS)R;6E>dYB*0_kWbzMdU}_6cW^Ue12Ll0HaB-6DhzGZBHt$k6h$gaQ12x6I8vF zv7x}f-;AWjG{#-26=l~w_0`ry76)^m|6>II8=H$t)86f`$4el66qVK#*;}r=x4#XO zM5azo?+LbSDs}mssTJ@}+o?neQb;`08(qEISt)jcBgvXWKe65_V}_UvJc?P8&KcO> z)_ui0J0$!r`XCl{Bsh3Ni=%=v-1j{DXDZ2tPKKLM3WXr(ns`bSx7TH!!h4?yF7-3> z%$1cfvpk68mZ0lsXI}z4#gtKnQe#-RlYg~eEf7cZlTI_q(&bj1Iv^c3HwlW{BMTx$ zl!gURFCBYJqN~bc##FCIP1J?IW$e`j5(G zfEFwLqKD>$e2DL3;T7gxy=uitaex0R`)3`=Vu$rbBFF4!h)Bg#>|M#fZfw;r zEU32x`TGVB0UUnac5OFFvdC!XLKMwHXX0_^HTAD7$0onSckg4ljY6O%&#GT93;C7= zA7MVKjC6=P`t_8V+j7_TI3tEIxeL$G)_fsh_67b2F5d>rCU4X8ZnUWB7=%CZD)^S ze&WU@Q6SM$Sg!)4lFK>|IYmukUpxT28`&-RuJfLU0??=y=`cWYv#DOkOG4?OUfsAU zdIIRtQ*8zUNs~`Vp?))A>Hu+8fG*%FKrW;0)lss29(?i`lA{;|nLMWXtfVi)c}>&) zHOG&q5sU0HHu9Y6|mtmAKLN2C+TF<)pB(&=H2S<$ot64RD_mvs@Q#;qa zRG&4jDA!|FH-Ruv*aRoB+8(6+7yVHZUp#`m$ zIzNBC5hcPfWQHx|6XwW?F_!iZxh{PWlk(zZbm12LSS;HyT(RDL z5st8#YxrCe`h-E^7gaqaG3S`nXQXnXDbmQQ>$QjJejLy8o&cn=*8lmU#K#4`s)CiE z%h?hX&7HoB6Y-&loKj2e5xXz&m}`J3PXY~ag!>&BLAwHXpF%C-DWvh28m9l^%NnlV z?&ysOH?1-2qUiHX=YjQ&Sc-|D#X8&UP>c7t0RViT{{l7X`Ovs_5RoJfo!)u74 zQ|Sglq&pP`K|qj_M!KcDV+a8ODGBKkqy!|SVMyuj?vn22S>E^l?f-t}^?Ea(E6?-z z9>--0B07h`;1NvG+f}2j>^^z?UrI0g2znSXj#rN%M*y z$ChioA3~EOzzbplfuk7y%;c?F=OFE)VajYT1K-4}fDjiHvwR;7X+%KK=J+^)gSmZc z9h)HOk#xL#Bc7C&g6h3@;N7Q7Gps6Jc(2PPL!4GjgzN?>rSG>XS`(bKw=B&4a>31+ zG8a*L7)E&+P4{9bX4yV(q>(sCGchBU9apVHU_B+HlK{nTx;VP>)mK*`#I)iscgWoj zuPSIvVPh$^OSNb8gGa})D)83~td^dc3BmcxU8oV??!FBIVq|vifN4rmL!fzmUu3a5 z(CNwe3YgRtqktd?agifWBB-H;I@?~@u*8#qf18_GR|1xGGScQ$39o!}=+KmIv2_OC zufv<)d`;+0c%kZ8(?}r>zf#=c=cdl`k_%=%jL70V$Q8vp`O|i7^r~II(xB0OGiU>4 z^ZeViY92#JX$9D9bdsBv1-Hj|q<-eOrUEYxOw|n?9i+oN*{ZI_{mb2X&Q38|OFV$c zk=F#z;R8IOb2ohrQG@=W|9hvt+fqpc$fa<|gVI_Z-P&XAoeX5>aQ_Id;|)=w--}1> zl4}AJDPA;qHh6NrGZN;EHcXg`)q-i2QBOlouxS_+6i=%|B#93;IvV>$5bkoH@kinC#F$~v# zZ{GlREp;Fx?%rdB7^_q-w^1AZwBL_(nmXCZDBiwhT{^)#6gpkne6G+Aq84SqH%4jX zL+4})%OrrML8-VWZ?7xS^S5`orCQgQG?cKcLx3ndlMq+n_mP$!g=Kl&`8rM z$0!lw!}~c!b2F|R*`C6;?m*`oMo{QlPx}dhiBBR$R>)kU9`V)Lsk>fIo_SfHQeoSb ztod>&)eN#Choz*g*wQJ28j{GV- zg1@%e(DxS`JA-8L`8FwOXc$MEXVHQT&qg-uu7$beCVVwRgMqE|n@Z11VjaFq`?Y%t z>}_J{&N0IR#?kclQpzp*ddIxn)5k!}8^+=PIbx!PLf+yBNx7!Lc{DW4IVi86T_#i+ zEPJ^49ri~JKW-JcIhNwjOt>(WKWFAQ74*cB!oXDL|Hc3GS=d&g42AOVz8W(uAwz+! zyk-mwea@>_f0>>sggZrC28Fqlx!=vfN2L!E3TdAyf5`p&6TbbHh&=Ev-G^tnfW6GM zi&|)|XxUstNj8llneQ@@RzcR#OLupGt%3=lZc}I2FbkP82CNe+*R1rG;+p|tlbu_@_;3oCq$ zo#xzB{q`RQXY1RyipN!2ADVIAG0^=|C{RMU(-wt&Y(x?g8sbODHiooM7&=7Fn+cZt z81@Cdk>ufVo_!s8_G-@+6RKIFD{hHJ+9Va}(5^k0fmA!J59&Ri7yq@bKX(pk;goxP zln%T`wmdkXvAw~%amSIzn8(d&>L#)dc(3S;S^%?Hs2v@-{y!UAyub8&`6^&Wg2RZ5 z(k|uJcGgB{k=b9?+<&`8&`L{JNa}(&9nS_wjxT*f0TG@}_cueh4KdUOizt?3#f7q%+1xXSVlVfs)dfbF` zVo1LJA{gv=`rW)bup^rOwv#QWTrMN%J`p||=r-D`TBELv5)loFpN)zDDDu-9aFm56 zX9*4WB7S`W#7^gn0cEyg$a10MDrPgZJbxAsDEYg|R~qh~go{jS61jtBSX9bX8g4dq zlKsI3%KUpK1i7ScgUup|{~H$9jnr8LK%G(g7bx!zK#cLSXzlXBBhNd{^Ck7o`t0WL zujc)1GwaAZG){2@8DFm!XA=Z8!1(YhxgdE(j_MMX9SL47PsBfK$GplBTDsw23m^Iq zX|H%?bAIy#T2}IscAJkBSt`u|P44QeDR?R3OT2%*_tcrrS$f2!&q|9jQ)S>KVan2Z znJxJJ83t>i2c=XBSHvrL`jZo7{=`lr(v@m?l_sZ(QWdIMp^r94Fmi^u{I0IhsbedRwr9+l&(mW& z{Ma4-8F5HF(}`emzL-ANcn8s|uq;S<;uBE(aXumBxoDkP4vf&IqCH;g7UMR6eOssu zT~bpn8c`%(;VaF^F~&Upw2fhS&WwXH&2=?T1aBjo90W{cI^}>&)PoGf(ic`+LYUS2 zcfm*8u#cUi7rQ92@9~NM4h7fFtQd_tq zb0zNsxtB80s7K}X9FAg&Q79SIYjT}OMIMsa@V(u{LdG@2zURW#2Y1k4bH35v*kZ2U zS;7z+_&6PCk;s&~^i=zitPQ7Pww7$a!|>>SWNt8|VzCv)5+psr;a2y-^GpwIeMoo` zcoE1Ab0LOOKQFD~DOJl|8h!%~&9d}D&ZkMtat~+eU!L?;mdi_{wxeH-^r3~Z*kkZ!2BL_%Mztg0R!+#k7dnu)! z)m*?rFTsaz)Hd_!eP9!%o~AuP8x6v)xeJQ&*=rs}C9`E4GdR2}$4J=cbq9}5JRs(h z!|ISq`5q2E%fkwT=2UEJ;kx2kkSxH_2S4Y-xbvit&Ofr(i)0RckC$Ai{DkHVr&M{{ zflSmX48{nPs^@>WN|IcVmvYqzD3GJ@fzy11Qw+xlpujC@bNV{(*E07E*|y*;IgH+L zxsC`O7;;LvOw;1V&h@LXfpvf&M_3SoOg-;^w{z-@Wza2*WOqGTLzSELxLfexXdXss zQ~y*5h-Y$1@chU6@X>vfRkkB2-U~t&?0@o}qBYqLRi=VPauiJ^ss&j4O`+$>fSrz5a~1VkxIz7Fp`Rlw~NXp(mS0GsC-yg^qPbxn6?~HgshpPP3Iv za*OWdo3YDH<|b$$DY%$@kbiOr9en8hWl<4CO_d+c=R3rH!9|%>o7p6Wm@SWg24BUK z^9Ca8%AznQK`R3B^*c{4xKy|Y!SV|4g)yA}^8FoPiv9rP{uZsl^s23YDA@I5`BKbt z&}?6(ErOyk-tgeGC^~%Y@FsVOZF^4i8J8hxnBA)CF`;rVW2v>JZsDb6JD^YOyX^JL zENlR?Q zG7b77T6i&1GLH>xtRQdgjB1dEf-G74jh`59eg8MJ{?*Tej}L#%e9a;&Ae7efEu7Us z9{Sh%`e=301_cF8Vv^!(NRpDu`ND*hZ)gQshF-*GQC5|`_=&Xl1VUd!Nv>rF1xadnfUJ@{Sl%KCVHv)kbxKC6(ZG6rA#a&B77&|S2V63nN=HaE-D=~ z|6H^Cq(HPsXlR%;em^gjiYbO9enrk#*^g6CZ5PievqDavJ+%G@3voN3Ean>NqKlNg z=}{nr`1Qw*vk8YsVJBY-oROzQ{w9RXxm1!=g|wW^4GBS6dmIBSKQVj7{$O@iQtm$J z@7V6)oyh;O`Xdrk7{wW{m^HokCh8$;&-u(fK6k5T+d9pj68(fz#*7hwyh>d_#|Y}S@mx--kwxIcrhz_|a?P9048X+nodKX~Qq!Wgkj^*x(@mEf6ZVls9CBrJ`Y8z@_dqoO zNYhF6P!YKZ*E%cRgy^4vL{-(!KkbF;S@E(59uRqA3bc-%-3tg$MCdC2MPXM$;s?}; zQs(v=<9brmH?OE3q8N`{BmNGp3xrybtEZ3;qd?Wmd`fayNmIj!+o01kKdv<-FzEHz zR@6Q4wY|Dx@s?`!9-rBBo)uE=QpDii3xM`9J($*;is|09DAs=$f)N}x@Ys1`_eW1V zT^moai@ogjo}SBxwVA5~**vD_+KKikYnv&2*+}p7?ruSOJ)>oLkWnc1+Vh|bia=Y% zIp(JA>lN<9un)7j2prftIK(SzzwZ$qJGGbW@qSaoHUUwhj_;EiHgkkW823;?JoP zzFvo53{&NE-6Axeen;tRYB(ueME$<#6UuAT$jf}h;IbvHXwMQPwJSGp4P@gS-X+S! z-^MuCUNto28E!w{r7(Xz8dAW1+#(sAIraP@SK@PaZaWF^@m@1-%K@>*wm{rd~&D2Gv}* zK!OC-AHJ{tH7pNDalEweO4x+PT*dr~p)myWZPBmyZT9z%hwUW_J}i4|i?f|YAhhra zR>QP#rfJADU-$9&NUDbn6Lg@B=hyq;j{kR73N;R7FG0Tq7EFoiSstU29+H9lJ+d~X zk-e~xLlj>y=>vTJ$%`kYYha3@77FXWx(5t{_ScH{q$D8^dK@{R|!-B0OOM7RBXLh|E#S>W!Xo|BFr znrTnQ$xjeTi=-QMAhlC!7^OXB@22qjy|(eRpt;%EyzdJvv7aNU{yAwq`Cg2DlkzKD zk*C;AC%e_6$_y2K8yYIKK?%~mx8U6T(6tDq>Aqdtv)7bqZ>pkL5njmZs^xIip$3wF zPA7D`1cLbM@Dj;j_M>5gS!`c9ds7@NbxPT7Ur<`w z3w-AYz3L%iDCP@264p2J1CbFa&N|H6ALQIRlorpzB&0@2*D-m7F#PY+#Mv1ram_aT z5+|Rg^rMTYRYQVp&nS22DB|6>*=gYBCdp>?2H!bsTaRP=HLPG42{~zPxwN7qa{8mN zsWS)76HD49l8uW!~NG&gx6S|xuvyU&fI@n&lOH&}6^VSK=9t7HEL!4ED7fNQNq2A3<1msM`Oojy3H)jD<)7AJ4p z7GP-vw{Iv7E$B%2`R3)u*`rEVdAlSZlH;5&*gS;VVhh#Ah64VBVwjHyDL~`gz7lJ1?yO* zq+@#_gSOEdg5FNZr8DnVdk}IApyr@fWRi63SrRh+*2@$x}zU z>j$3sS1;mAJcM#!QR%tZ!Bu@^OX{W4GeFBvF6?DzGwdmv8CN^hY@&k=9qMPt?KG_h9ergG0yj8t3}w z!zm=PC>N)u`_pLCfRLo!OifysbT1)b9eb}1cOCkHja=@#HH|U6*}%dHVg6w@lXX#C z2l=j-9HXrGLqaTWF3v3JA(`7-yP<-$ZB=B zFIU)Wt6mi4`n8yIz?57N(1d=pswJCl{}Urz=Mgi3mB=x4)!&Fgu6D9>=m=%0!f$cF zGIihj3&dI5j1AXT6kb{gx^ba78Aq_jt9{~RGW*@@ON7}dXEjR#IOzMn`hSN%%DRQr z2vK~YT*-R^?6S|VUc+BLcYjN>e@+%su2UOeQ-U44gf#X&o-@EE`>tRbmhp87>#Z=G zV^V?x8kpfLA%@qOqnntKm8Pn}A_hZWS_6rCybvvEV`Q?N$OL<&we^oLU|p)W3oT)4#<;Ex@}E_R1K-7&2TDY#O)`5+ z)NM}}#mg+n7cl9(FZ(UWt+!4SwuJ!=8$A6S1M|lv-223kC_T)dUz|b_>wPje5u&C?HG&0pA z6Ad10r?P{&S{l>C7qg!3FLWCm9{WCIA_-)yl!J^ge$g1|dh|RkYE;U}tl3@=uRB(h z`6tQI)QONk1EEzk2b_%wXr4pF;PQ|20wzY8DV2O@gVe54l9rd>- z$WIxk_0g`n#62nCE<#HieBN4hv15$gkLNo+xj)6qbkq81D}HDap zd9|G*VaK|1kojigBU};$j*jNk{zrzF*=iAd{~xAR`-Q>RFUy7umDam*LStoyl9L3o zyVUAW4NwTeKYGFFyP1<|dBL{X+YBF$AN{qviYVt2cJ!fr4kaJJdehmV>nE)+D<&b> zu#LE;Y`bfO-ms(O<=b%bLV~100~W*IlY-jXr8l9AeONo&6!?l3Ok_df2gh-eK> z(!y&*XpcH1USKI?B!M=0thrIoVEOAgTWPWx#cF`_frq3&b_cBl?wuKY~5z75f`4W0_ORA;#)`I>So zg6`1)KeMS@b$-W<0cbv=HGMO3T*1L{Zo%;M^|qkk$)9Oy{LyjV!7UzUhIPY_9l{kR zU~X7a^p|3LphVzS6!;4*{#>k>rvjKnb@4&Ds-60r#FRyFi15CA0dFEih*g6V5d`^$ z4K})IG^MCTMv%X1lFQF2Zw!#T8!>PDZ=5ibMt~R{wv4qvB7@honLZMsFU-y-MAty19|ouurGqr~6rYP7xw5rtdj1Zi5$xVJo% z*86+4BF+uV${N5k#V>o5mFR6&-ce__(D)mdxEa7-H0<+ppC%%FhQhbbF~k}?)VLX3 zsjBYno%inT0|u}b1BS=QeT;-%Z@x2OEV+_BmQx^?>m}o%F{l8d0vZh|wKvZ;sNk!Y za&tiBu{%@gyM_5uBxN>=;^-+i(aZ)d48!9WWEkaL9s(e@%7bdC_WFzh$Vkvl|I8Gz zr?2Q@kIY;I=8xSlv3gcx|le$2!Z8l*~9_CWoa97c+(Lj?A-Pv`Q!NIfoJx{h`Rpm^9F%Z2rbALK|vS z{YvKA7e@}us~aipLk}M%k<+1?+bI)r=|RFQQ%TA>~&E zpIJQxh+)@XSAbj&sT$}JUo9v>;UR;{lGGkDb9f~fxuyWF)aq}nz|i#{DwLzUo{0~x zQeM_xO_?vl>(w1VRopwpta<>5xG^_fuL{|ID|XuC7u-tK{sDTA)HRq>i4RY2Pjl8^ ze(u$BXEBz#thp()g#k)OmT6o!@bS$=Tun+tR1?FlvY6Oqgi&`SKI|_Ku{sx`FGLWI|t!L5m5`8KFDX ze}3sdqInoLaB_CAojy@={%sF1c3zpdq!Z3Jd^0H^yZbb+gMuz6_T%SA>#X2D-V+wf z$>0wH6}&N49gs}Npt9xZInRCGaN299)(kd5S)W%G^8LE#e-ExJmZG!Nel#`ypzgq) z-67^w4i%scEr&_BWe08Ujgu^|CVe{Cd@+0|TTS)rKKs2=biNoFmv&P>_WT z8+WrKs`^>pf?a-~A;%719n2F0Ou~(&fp5@;1t3f7_xmsLnB+SgsQ6T!RzP2PoFia; zHz%T7Cu2L5Aa z;=BH#!0x%9NF-}sKPKTI81dQZYU8H09_z^W1JcpBQrU0Ki(N`2M$e2BSr%pyOrhLH zlLF42$YL4)AJeTi-FNlO?K%X+w`M4*p^~Qq50HFzF2grkpTaF9d|teNh;ZDy!j)qkD_HFPJxr9Yeq zR^54o<_!FeEf2>AU(mNtViApQh~crpze zR0GpE*R1UKG*2bKtJY8MN;(_(6iSO;mQ?T0Q6x+LoA+)_x2j*Xl+C$xHG+few=Uk^ z64_;z*T1}j-uU9z312)Z;2UOWeNGb(b%_I{GZLd%F=+jaNa1G6J0ft`gRPek^bTah zcPSz)(@O(&ELm401%{{!u9+m9kf*+u{y57G!SVb+=}wm}Tt)JqS)!|WV~==-W*Fpt zq(#agPuXiiffAe8J2LoI1V<{Q>&+h`WDnvx0i-NhCSdt|JS`m?9iR0Zi*(x;?nJTQ z?d1i2UE66MVRn0`zo+C#eYkwy4fAtN62EOj11bC-kZ>11qzrTIH$?gF&PHv+FL6@p z{UJOf$lT<9?A@FpBBy*fdOcMvm38P!NrOVNL(Bx%vHnj4!A5c{XZXv9iW6KN+aS z$^-K)>!iVo^in9vK?(eUZX&;e0irHaa!6!<^a1Ab&LgwqId;~)fD|bbLhS-#+U!$90l$4t z6bH#DCUmDnI7bw#mANN7p1V9Nm7~VN`u$x@NKxKk1`St{b=&ii(^75q$vQuNh!0uD zQy0R>>Nl!j=*C@Pq2u|L7?lz`^G|xtzFCLm^?k`pX-QR*$Dw!Zw1qas%dZ@DYydf7 z@I!TtL0k-}pwZXO13c5BSCiK6GLm;lEp__V*-B2bOK>H%wOt}aHif2$66gPS=pW|x z%O=Y*hy>`;8qi`!p>BK`utfD-BB5zCpt)hxWc{2DQ8!O@LoI>iF)o+|fY-h_`Jga* z>U9*wUsQ1PZYjoBQbHO^~urP|7ZSq60?Ln@0nA1+()U`7^w#&~|#f z7C+^_@Jtxnl592jjTf!~TUuf;h`4_8TK{}+*96r}C@7EMhvPY4BWv}mS(YpCy%i#F zVnhbbs0HNera=LXRIDTx8KE{dpb9ei(n*tyTL)m&Rs$n0d|zBo8)=zl~N0>af?)D9HQHVg9+l z?k3+6)gJl&2w`K3>kC(J64O}S-fky{CK=@*b6Q60Z|I?NPV(=E+Us7pD=z9{NSx1a zdD1|}Sdz@_c4&FmkqAdhQ~$orb<2aO>)8785kr#zzY)b)^~ z|J|hCn8n4G#2GEeFSk}3e2Biq6~d|MKMXf04?$E6OjQiva+N+6^Fc9cnlEl}L$RZ1 z=68Y02@k#jlPc?kz7(gnXxYutuk=DDB#RHU)?#PQYNwy|&?VgG(L9wP+8=%=2|n}% z$WknT=iIyOu-8HtHj6QlHbTx7hVF^M`Z7fVp9h2B_%%%WGO!)i+qa^#(U{G$db4os zH5v1^Y@5@jaRvt?^X!p3;%D_5sf6BCNK4K%+Z(98ycK0tCugLZlQN%+IUzO=7kH3K zMMY1dtXhMQui)7&d^YIIUQ3pc7%``)8zL6VlDXvCB7LwvQ5c??1WrhVNev4MR}6sC znGhR*j76NCq<*1O44z~6`vtJX_qG|GR?`v3P-;t>0isJxdWqCc&NiY}1LE0)1qi!M zi3|6{0Rc{}@mCD|KNzJrFZEWdv2MD8ABY{W<5irX#gJMF4js2Q3F(HHN8G0QBuc*~ zLl~d%kf$zv+PuAEP1cYYvS6*ox`M>F^Tb|4x>p1?=p3uR2*<2hJ67_CY<%JI?@fn- zS^`mWcy?b^*3|tI2{M^RA6&Q(=Ey%7^>Jg);D1^#TjJNW7d0vp8y~LgBSVn0&Hqev z#@KLvOQ=SVyOGjfwZ%UU{HTg}9rM9cvnzL@TTk2RBQO!B`b2YUSqpp>l#GPV)3&vX zjrWX-YKHy-x!~o3b-q1jH`TjDg%P{;(s!3ayTjzBFN%|aP`k}V`B#`EUBtwxGLkKw zv2(_2*`vkO3qCekZ1HPm{0O2oRU?9U#?tvtdLXso-W6V5JfwgIf<5w(E>aUhh)+YE z45S;031UT5%~aoqon;SU!;#lV)>1vSc@At4kzsKa+4f5TW!9yifD?|d62AIYttM+A zDXfxXi-Zu&Uq=Qiga@52meyWF8l&rQP-C(beTa^#9wp#G`rEc0KtaF4BYXP0P!3o4bYn5>9fhAGE*Hy%1p$ z7egzm;`ZGU$B1XC`1x+^0eIo{{!_?*OGq25#Y!?fI#0dUOYOQCA?^yj{+9*wt$)py zXe0_5MY}01Ytqa1yrT1o*B;K^eki?0^MR+LdzgrCF8gsUs4_!G5hC8Rqx=_G8i93HZJPPF6v}z5?9diYXHsBRPY0 zq}x(!16-ErHbK_^RInbM#aAXV!m;;EaZSt0Z7Mw5a`MSY1_o&B6Na9ZsdJf{dkkeU z069M$09&CBZIHq8NsCx$_vwDAarc`aGqEv_z&-$qG*O=I zwpgd)TVlB?NV~T^8@PUrx@4*s*r5_TV3IE}ZLNQJO@*4*u*Vv~-?XT> zz^Mubbsm(gQ3<&)T-xmt)DCYcW(#B7D14bG{XoH@aJ0Q*f@VgG{GE5_xGJ(ap6tb- ziiFZ_17IyKe2EKltXRP2+Ejce2#odF?P|Q78Mzi35G_sXQU}w#CdB86U>{#d0@CV{ z)R5(wf%C?q27|g`u#1fJw8;>BDMuk_{VCuX?Fp=jMerhdc-9MOy4IlmJ5YGAx<9dl zoyvP~AA7=v*&~$v1ZSj9jNv5-B1Uwj4Am^qv6+W{kJv8lDf|8!i; z6YlL(7C%95+pN_3k=q@r)2JR+>A}sr{#Q_)!cYyCgU>N5bI24a0|9kgReNY)k85UBB7Ra*sdf1)laFjm3Tlj}gIM>i= z(nYq2!YrxMd?frubYmew>X;+ym@PNZ)cuFkl|+NLJwjyhy7 z%szvxe4-8`SYxV*&OuxUhD^&QxJ>P@3bIZO zBT0TrVHj>H#ZB<8etpT<`?#Sgnv9iv8@^O7-4-W6#@}D|`ZVuBKR+*qkFP(^7!$og z=(zv{G?>H{eV{-tC`t3Bs9Z5G>NGCKF;MH9PZLEYC%M&D(AYkuHPu3F(H4aZukwTc znJi^@a^yNXH_#8`zlH~w-d9g7nMDHjIKGy*gL`M@_}iGY_gpO2JkM|qw;2RkzC4aF1u-BXlj~fX0&7=yu zmKLtDS!lAzvFnJ%SeR)dE5Vpf z41WjAeQDG$X8y%sVc+IjmV@1=q22U$KC}#|C zC%cf5UU37Bt-Ct;vxP5yLIY9e9NPU}^4Y zOT_r^O<45j?d$~0TW{r10*g5mxf&O7mk33rRnzqXpCj+6u_9f52^qkU4B!*PuOrE z(oOiM1aJ%asV0~w*L+&N%;^)*{cQ>!{$f375L1dD1cDsw%^Yxx-`0%hUgCOP>NBke zZ@49%wSpQ-UkPJ*0rNdf(Kf8~;Iw{9ldFUDL1S2f>w<_Xcnm%=S?h#5Z)ADMH*Yu3 z;WtK(5kYNw^$l!>3EqF&c_KSYs*z_*OxoD_=9odb*O4P9-OtgN+|GxxrlrAGV-9z?jev0d*Kf&T!w{h}mUE)N4A zA7J{<1)visNaZ?R1zWbs=b#*Il8Hb(b+`D2rLc{FFs9&$C=f+KcjG}vU#qvoi&uWf zo`s>;3EMhNjt5DY-$#bHF^qR_KF~@%Q$@4vl{c%OIQVkXNl4Jif?vJenu);UYelt` zBb|Wp5pZv;J{&8#9T9H9Q*u~C;SM@s(lFKfUk3jrPrnV&qZM$795W6=66gP9iisZ$ z?x%_^lNnJwzkU5aqTt}ThAt;*%Ih&&;&c92Hb9PFjC8y$Nsbohjx96PM!az zPGYt}5$w-R409d*f#r@FhgbI)augY{?vTQtkZ0siq|}w)d#Y;$Nbc0Fd`wz-y(M}r z+?2g^JJB5d6DNVm`|QLMT@KBw=l%KP*ccbf)sXXL!yrc<)sS8~>Tr=sFvPIGdRiRv zuB?_!9l4+qtio}8;LGxq-GI2$jdXs=ig4@rJx7<-6j7q zIU*>TaRpdvbi84LH$I~RhN{Cd(Uy~UW~ISoNwCTJt<8QfIgfwkv^ja-NfZ{E7r@cn zXLMB2s~)%+<$q17S79te1Av}5o>(rC2(z0J;1AH}sev0Ffw!o5{$2rToO?iA?) zzD<|-xd#l9Pp8uSH_!eXz$98GTGz{;$1kKa=RJFilo3@eCt|pYd)XI2V9GVlH;%mV zdS`buDr-+Uin$o^EBAZuJsCdJ3ab-B8{8Owv{rs_=y#4p`!f+&J1PKwD_tn=&0$dg=~oCHG+y!| zG!IAOz?o9R9KJR{NOH@Cc$V51zM(AG*ZGMo(B%4 zHY2h;MIfJWiYPScd27gSFRF5*+!~cO;+25v(bbsUexx0TbC$nE^;<`F+B?*-+XqRB z&oSo7;lFbO2`%B`MuuVd?wt`1D|Kxir+mz$Mxr11C<$++yE^$o%U@*BT%{xBL8;cM zz%V_Yg)fD&;1xAI)tD=-SgZVS6ANFY(|WZ1Fnj65>VvVSWZn+7udkfMM@4sSW!y+s zqL-k-T#v3LpD_w1xgx(hHtm!-q!=U?l8sPi}MG%?vQuU4sP_mz@bu!XTctx@R zVyNumB1j28hpkj$N{5+7vSq!$vziQrp5=VPU{?9Us)-|-^o$fYEV>ix=!oHu%AeEn z-Tl8)Cylg@S_Lo4(awhOZn=+Cxt@g;8{-*Hc@$^qgmAij76#LA737eqn#*@*D0s^7 z&f*iAai%evwnEgDzZB=K>Sd`Z=$a6MgXQfxtQy}-DpGMGw_m_| z*?5k*uu#hYp?v2X4P}`8a{y` zYvbc$JHAKlGj~G?6J{%S?}vpII2!l@R6FuRs&QsF^zUEyC9m?Q;TDSzS;G|J58z8T z*y36%>XtVbJ|8*gJ$w%&S+&u4VX+t01V#{P?<+qnx@q>9(9Se6f*#e^4hz z)J&poyh(qYU|?lMW^QV>h6Qo1LHr0D3cBjL>r>Xh z5_S7m@2>IDmr>z5hgZ$2F93Ru)GpL_s50s^8p-g=%D0kha%bN$!4!AShTbyTM@jkh zplC$7Vk0Am{} z@e=GqQou$;{u%&NiNt;oW!K6jZn(u(CnhjtI;A|}VEv=e@zc#(M>saxvfc@ha?Y3CgmB0V`|2|78bF*GI93F zm$BZmx96k!VXbC&EBDRxMY7sr)27ZkZZPHZVjh?#`Dloc+dXQWsPDXz-6e4oE9XAb zCnO|4LJCVUuJ@?ZCDp_v#2$(K&Kf3Ipy+x}S@7Sz(($U9^cT}Qbi8@O*`Y1fu*52o z=K^xAOdadp*BaoQLO1PzcOsA#1mUpVM37KDu3srK5}&`PJN=o;RJ-B4VtDhh zfNa9#P3jIUP4D8#1UMb*>DmlG1wyMtIcB*)Qw>{hKesOI!DbS~&g9T&;dDbYPoZSPmDy19&-ok>5f}s*W zwTJE!m9b2toOB8ez)3AoD!><#*7KvIvnJbP^JiEayYgD9-4C5~fU$>LrH5GtI*Ary zXku17d${gS)D>9kVt0E;Ba)8Q4E?2aX7RF{WZiKGZQhjYBRHLqKZBia&Tn-{{_6cp z-?%ApHF~S_#o*GfkVMoTFkb7uBr>329vU{tK&G~I3Fvcak%RbBqQo=Y(c(ci+%MiL0(57*ZW-Vh?I}aR(G_Cg z`dt;X?|D6~_sG&EbTzFPek`X4I8Nv)v3%0>neLll}8pd(LKI zPJj*xTK_p2+G@5@Z~^wT*-MgtdQ{GD=Q^+!jyY}4vwO?tvy%EzAQfEdbKOsdM=6&# zPm$5xjh_C1zke?aqJJzO+Kis|C8O6tQx_HqseP11`hQ7_qwW63Y2r_LKy|ltDTdlA z&Tpw!iAEYqzQjEVB_w+;*%!dg-B#Xyg)=?4lzHdC^7Ln)e**%k)f2~hq8n*MNh-`c zz6F;}Tr${5MD5##*o>>tOZ3`nTR+Bv^*KY?fwBPQk>KHzOCS6%I71X-sO_F62JH&W z4;OB;3lI1{MJwSc=>bx8l8G`4xj5cmieYtj20F7e2V0AKnU@?WEmK@Y8qC8A92&g9 zhWsU77yjB0<%&BaiAtb@%R-|MczQ6?3nkA4yBbf=gTs3R3oy7#2Cg zn0&R^sNmsPCMuK{#NW}Gnbq;7!uj4vDSr<~4INQKdUx9u4`y)`jM zZrgGmyv|sA-`xU-6>OW9f1Y6EdDu)Cg3NFY!`E1t%jB9=2{YUwZv8 zlWZ51v&G894pAl#ccYCsjnyq>r)@#QOlzt+?Sf!-& zP6yLiY=0X<=vMIy{cm3<6k%pZ!ZD)P8q@*CiQaYno&hLy(srU8xfoA$ou&2@0(Fq9~T*XGnSF$Fe#O&v>lzdME-|SM-)4k+|-=Zdh z@9O!7^+9SiR_p>z+s~Zi&?a@lHQwSDtT5I9V<7$-8Q~?anrQ%xpXTfMjvA zfD+yE=9?wZOLo(p370;B)&NRgbxuN}VteZ*wT!1YljvHmBG(~16a_P5;sa%rPv_&# z*WW7do$%J-!=GN%D(lWv|K0D2?*Bybu`axtq{QdBXHgGIvdMk}=F63xH0&R_SfB;Q zpW7*$O+i(eiM5Z2hO6<##b?SGnGGeLsq=HPo1 zFf%wUC>x@7kItiBr=B!LLWI?X>8P^aDHF5S!AK*zU0ZHj@q2(0+V=QXCwwm01B)|k z(wR-ULV`MBBTsa4Av(H)Du#dK{f6zbt#-;NZ3FWuM^b8N4KGKa`-Ry!evwj;8YRe^3nQ`H)DyBh`&kw&D1AqDA@ z?x9Nor367xDT$%GhDJa^Bt*KqyWhF)=ee)X|EAV zO9iV*2i~gSY>X%!dR2gWN1SYH#RV65^+~IYU{=LI(yW}P``U(ABpu=bGwUmjC7Ro> zYO*C&X4m;*!_UbqCb%1Y(VTSh>-I-|dsJGRP4sw7)O_$zfR0(!1oYWx0N<|*76 zvwz>h1iRw%W=0(_kRwE6;Wi$#2bl1i;X`r>T7ib8FHO9OHMl=(g0U*<61k8o(^bN# zgY45b@Sh!t959n(rHQl>d%BscxjP+GbN;FqLUgfE3*_6r;ZBgdIB_%y{H)aqX=-<) zo$7)&n2GNTJF;Btbk@GMpZ&b}eZlCSU}R0_8)*k*VLALz^f?kDy^2c+E)P||>-ddi zMF#%qp?m7=%IWBc$z8yvE#?(XRTb8GtKOyVthPgV^MLOzBS>`L#Dd$A1|R=(ojS7m zs}7ugfvllYVAyFFT5^-Do*AbgTb{eyL)aOb;3wLe-YNTf_FL#b{Ojzw@-6BANV#on z$b>zId*wa!g6(xb2Gh-32X7k#-OBTq5;o$VyDu&yW6pDD0(xFyWrJU+*S+VI>0idC z8lO!s76E9`QMazyL*}ud*fZ2U7DZew7-(+ofgR9S|1uYhhGbYY(NLFR9d5$v3*)d+ zIGJJnsCVzW;u8Oh*5BJO4OoN97!UU8Aq`ChwKZnrGcEtPJ#}@%k!>;#{ZvB{0BV^9=LNpsj~9w4c6+6`{xpN7PbD%U7(y{Pcg4`*lX<EeU22AUmuZK{ z$}bgW@O%pLog$Dy+~RLOVAv~tr z*4{1c*1>-acW~ag3cgtDf1u-^<~j-E*Rk5*!*gqG$o$NTv|0f+p_MfD&C9$=4G&9# z{8cxa zxSGG}W@COD#77B}TQWq<_>`%GCJ1Ai37FOg_5daGw=IMVwuk--B_^82myIoZ?Vu_a zUbN!NZ2S`ZB za(C8)@xtE2>@}c%DnaGyZn(J}Tgb4{L`{t_7SVAUL#o$8_Znq@V0Bk}VkyM3B=8z5 z4an!DUzJ5ISS^aZTel+^7_6!%Cl26u)@~snePu1GeIACCT&yedVezQVn zPjEntoFb^)aF0u=1ujjU8Np5){&oC_*IIL6Y6(6oImG&b|^0#@yW{g#DylqIi5j}ij^@@C1Y_rYR16c5RnsXzUdlk5cOZ|>OVv% zcV~%aM<0q4D{&7dQGxl%D=^Q!fW3n;QPCvZM#Vj6%p7GM&m+pmRELcp@E4XAcg{L9 ztQ<-#>RT`z@k@6;WW8zv^RGqAkX7JHu|jp8*Y07@?;fubVdRq+R6yx2r4Js=B~1wwr4=&(I%rclP5HN*wq{5gz?Lyl=L?O{L_6r-|OB(;`*wkd1BI1-~aUo)y{ZF zr9j9lrM@0Z3MDp>$42KM(CwjgeVh07IN~!pz4>3>m8h^`G^+YIzNL#m;x5+jL=0P_OS-GPu`g!pxP=+YP6JmQgN=p5pP{1Kyq)&g{^?G{cA4h!ozpDc+Y;e$eQ4CROP_B+RJ8ElE@#q4N_qQ;@yaM2X8)M#PU z{c*Z84z^b3__kG^PHv6+f9#ZA-iDKlWeYn2d%!Z9wffO<(tQ%-^0{9dbi>PDJ) zhkCK0{uUGi@2{yL0w?Q5qd+i?nLf}z*{v@~H94E(44fC37=ubTJ-65p=Uj>EK=`%_ z9OGbW-+mnFAFNfU++Q6O=x|2Ufb%Eb$ zEwCvMt)Mz6Rld(`vXSa{!Q)LIGa~LbcI?4uMHVl=IZa1bBQNR3H|X%GS1QKfi%zKS8i;Bk!2c@B z4^D_CHEe1a_3Jzw;Sc>^c|)?qF$^7u8VTX3_MW9EU7~7^->!bc^Nu6SJ*OBK(@U4` zX&sl6`KsLM+?vAFAt?Nm|L@P%R#yQ16sO{m5t8Gbk4zB4vemmtjNKyQv=42l< z@K|b)4nDcG5!Y@B%d!=LnF%(tqb7TP?m-S1)(=@wsG*n+AQ;j*ie7}raoex>zIoY} zkcGX@{KEw*J(irsMU{Lx^g(>C{(=wm1h$Sv7=4O2!4!>nSO8hUuXqa>1+0z4W!^^Tu>(e9dCAyK*8M3^$GUdK>Tb z>w>=1322b2E@4=#V^FzgH|Po%#E4UC6Ez63%v+ZMgXQeJLIjaTBe_1e{IK+k3IgVk z&=^v@p2sL;EVp(#4pZ&kdH3{5SYDZ5XqvE_fA6h{WR4Jg3M2{h@+B?y(jNK+2+Jrfpz zQlDw2$ioE9%m1Rn7kBt_+15ic!V&w9Ka7s?#xD(U8ek5Jy1FdkK#;WAod#cXKKPhKZ@|; z6bN53-0ZopU)_EuKGoD2w0g;W6<(33bSC!*NykTN=xrO%!hb|3u{E%D67dJu=hZ8( zXtvrgLku}0dY_1K{0xxiS4jfD5w4U;#?KDi-&n%L-Po%tQG0*(gSL-p z)`BKw7gRn`BJcg z_btUCdxYu9N8&#lpJ@z{P851D4feQ%-0ptRC{aS*2N$+GtPGP<+t=I*CqAhq$_CLR z@`2J@(_ynQQ?{AIq%D~5=4pQum#!GIY(4dpX5XouHLDVPrJ znlc@Kz0N`R^G^7fxhJ!JOqcMi{o~xpp0OPxd@Ftd-_B6y%G(XUB0Si0O<_sRm14{< z?$^J$SaSGtxbpLiCMKVkYg}}LV*IUVq&JeHFI8XjU<7T~_6=kAH2Jn{w^CRa-Kc4; zbP$^wj_=Os3&i};h-Ashc)<2VafaXL@7xAfN821uj~Tcf^671BQ@Ce81-^na%za!@ z10)zK)q=04a+!G&_UFjHZ%JwK^=ZI2Hb{!uwh5=3>_ype!%8E~ifLm?u#8vN)FFu@ za1{NQ66_@Nx()+XhIWAmb^g3^$FoiYphN>|V}y!wxv=N@?bA>uSA>y8+uClkO6 z%NOhk2X4FrdhNY@pfd_^daVA0if*9Plt!F!;pg0qAdr7Xc}<2YX|8NCIDfog|5tzH zJD*(GcI^?Ri5bG?Hd(8{2I2FGS+6F%;^SPGj8E8x^~KK-8ZU`!KX`gWl|wVG|F4Uuh-}VeN5zh&@SQ>$9W*rcEVj+?8cw<5_D2`+V$`QEZ%Y${>8 z`LjD^^Gc|E$kqz1M%!h)`>YJ4#l-q&bS^Fc_=@usa!9CgEY#x1H z36d1yD(QU)`jHx4Z4!|h27JI1zw76&NQly>datA*=6m=t$tu)iVYQVt1lt&E0Q<{| z4&ag;`Ts9WfO}lY-rv2z=>8Qpy6IP+Jb5ve_GW252w{FPFM`WUf`*zOs%5L@aoh7k%m#eOYM^f{LhqMb# zY0c0jt}yRh&v%eK{_Zr5YgsvLQZCz6)qC0|20U?<(@6BmC54F(UB#2sn!sogl+$d+ zU1AXfuO!y+V*U|0+rb7{pG;v3RQPmX19t?igT-c{8M(9R$g%)-*q-6UIN%uFX9MPV zLW3``qozlIKmC3T{p;$&2Gc3*uicsatT{AJyS)^Nn<;zJu_D z(Q+SGl_^=~n;%L3R<#>tSqVl>k9fOmZX}5^6~w6GV-XHYTW_}i1HFF*!lA3+w{~JYV_asxHh(E43~?PxOJL}g^^Z+ z1@v$EL9t|2nR(pA%VaKZGI2gv2R^+07&dC={q7P$_2kV#cOf>zBjf~Rg=nmMJhyu5 z;6Qm`AArcmT;QV$vnSD)LBoYn2@ovFd}EPN;IiqTBkurb|A*LT^<{! zG7II{ydU<-xu48OyziBWWksC)`@Mm*I8~HHI+3kmc|BBKh|0wY-Q6IJ(pUZdxd7~Y zI7mE?m>M~ZuE=C|P%NeYfk?EL5eJAqv!h*`@9_SgrOCB>m=D#D0sCR%O}X9p9KUg- zVBQdRbm=&CB^TAp7gMY+|u`(D!JQ4 zi<4jeDs3p_oZXd&3w;W$lF%M@^Ts8E&4X4FtIME&1h#erfWGPhQbg$6hjSPQ0mJHp zhUgk!Ul^J7oz=XOh{-43OBnT+XbBfc@fal(@N`b$hgD9@-PWLU!hH|F*$}|G-97`O z;%fG}0kALHD+ee=J)Q#{lr4`xEom+^IR7oWJkZauhB{UC`&hPXVc$`kI~F>E68DV_ zVcB?wo*;TpiqD9voQ1KV&6kVoVfZud=ohJ>2k-DSnrB_hYqW9S?-%}N}y$7&>q~)> z%rd3=hu`=4w@cY~i@sydzJR@8cLVgg3m5dqg_92y;D57&Irm|n%LBD6PhgGo!8uTy zFx^N>q3@Aij(87#sjI{Ft!b+l`X4{v?LKm|G+avae?foVseV#&qW>Ojuq6l z#BLkYWyh9?`=(LmJ|4UqrLxLVy%uicdE`CdFD6T16&D@wzGA6c#etRR|75Xv>oPzJcHdV3~k7)x&ZT9y9iUFGR>MC~tk zPN3?m6Cl9DQA`1=JeuPNlGsH_R=#wPhc}HIbY!M6043YmP529@lP@8t1<`kHfSkC@ zZOu0Q^3wM1gLh$nc;%wD369fD%p>wAk$#0|VOZQ@A&2`AS+x{rgHce_)@xvqC%YQ%!hnMM_;Qf}-zzS{^~TznsGK4Uxjhq@}uIFyz-`9!ggT;m>zPWzLa! z+hRv#=q_bd3;s4>R4epC&;}JBpcDpKQ|iOBJXKl)g6vgaGT_^)G5 zkCFr?g-a+AJd3{kA9d5W|7KnFig77M_r#=k>c#r8)bONK*AGk#e>CU<>u*Y;c5FE5 zXq<{Ojlg_?X_sCOxSsS3cQ)29u&;mk=vg@HH2pH=YC!5&z)Aw7-hE-=P+ApuL2Big ze@kHwA|0?!Rg(vq)<>GhZ()CK4l+9oAxJsIPYRWhT~bpAV&|;55wnwRBo1iEYa-qkf^sc2#6k;pQn{JF-VAk<#!hPPXu`CZmGL@;t-kjJ@U!VsiF8T}e5imU zYZcUKGw#DXa6f~UAC<)^Rbd>+w_N-Tlln%Z8$JG_Q;rQBtvd3x&MCiTQ+Q+AVUaqf2Pe2eDE`REZA**5YRc#L1g<(n72Wp<++E}bw+}ZMgRz63cWsW?j z&a#$0w|QRn%gn_}Bhz<-O#UW+s*;R|xx0z1Ko;my=q*xnBJASDP&94E<=upj<@m}9 z{5P-NYu1&0gxQFoz)ot?aA**I9-hvE27L1$c>n(YSsU3=b(fTVRAu~{MW-D0nGANV|dJ*e5Xi%hoKAO78~;wzN<@5249I zHrS-<;`|eX$cpx$dHgrve3IQ&dbr$2JoMST(+fkl^@j_yv)N9>Y0(a;822W>tB%v)o<2Yfik)vyEC8xF+2AmJs?L-J!8Lv8vMmIIY zW#^#Ouf>WVAY4_;x&EFSY=;aOV-i4mK0d@k-o3xB%;)2o3Mpf%#S`%r^-T_u-uPAS zl=wc>B8H^CryP;JoTWlB+{t>LABLfXhh7-`g8CzFBlA#-COpq1xOrSXbpb)izsA`jC6SgK>GzaMU=a;n=TFfN~!M|^~ zp@(@3QqPLM9%C$gHV0=H;ylY-FT(nQk!%Nf!xE=K{G;1iaEt(VXOIXVQcN5Jis@qB z0yLF_NI+jQIJoz~gie2EH=3|wgl=$e`h~(m$(C5F6hz2ytT>3D1by=9qWwbiJd%Lp ziOH8x5mHHz2w6A{?iZTX?m-tLpO9cH3F0RO8yFnchjb%v>;&~%b0pWJyoO1Fhd}Qm z+wGuw#38y=c0ct5HLmHfnk{E@Vv9?m&!WxfFxtLKV&yg#gUE z@$2n7lu8XL0b(t}U88w!&oktF4f+mujPw@|1fSZk=yZSSvKjN6tI{Fq* zW(eT=cmX49k5^H^|s6a0@ALN}6|Asg6+QU}_g)$@Y0dAi7e{0X@ zHQvsQCxb1?dvzC03_E)bwBN%*oBYbHJ?zoG;l6uG!Hd0 zJxY;BoJ%eHB09QQu)r7A{$i6`dt2|foyR;M`+-gUT7_(<*T`oOsf?^4pX$i|iWA&Q zG)Q^Mn}{{9_*i`4+lxs?WVgemvP`d`SW5U`yGxRErR1j{5VY4dw(>eYcpOB0Qqs=< zIE1cFaK)AQxEo85mSe;i)07ACxQ6fIAXZP31iM1uS=g5LjBRDln&~|L45Exj&5bKz=QLDNF8DaaM7PvH8--u?6Ge zwf3GjghbDx)X5i%j$-HCSx|1xYfq0nm_ky1nU^fKISG^QjE;qBc!vNHUHjC;lxx_M z3GreBCT4Kj=!6%7|C!8o@Oii!btgeR%Du;Xq78>UF>EC4rOtwVY)ugU#enUKgLvt* zOQ~HE0il!YvFp#j4xh-PAXg^?EKFxkXGj{edEmG!#Vp3^?43 zI?Fs@S6rGY4lYLZKkP()VyF<9_*@(4yvE&0NYJsU*Q=n1Y;M~vU5xXV`+RWcm-zd) zjn@bmSPdh@G*{~2P7|!7F`jkt3{h{c{31x0qHy*XGX#`$b|AK7X7lE`%u=p;<$yDb zx55;6PaF3fM#{wKqaSFGP8&Q`h~hW=sH=zC8WDJ3_g&O?!9F3iFOH*>3{@_u-c2&^ zWbHSH62SaYCyvtadQyem$HRp5ph^V-@skT^N0T*iDuu|{yD4TH<~6&E(V{ux2m8Of zLLPG5Zah4OGa*Yk$Po=sx#yz79FN5FaVwHa@cniIoKg2pbOEM)>i`8I)%^nt*kDsT zaB;Kxf(qTXQ6+;R>0j^hMgdP5wZKjC(z&-`srb3%$OacH5SAq(UTO+gBR;GMW*|ol>J+?akLBhh)CqY{c#uf(f5Bj zYZdR;(_LeavUO>U$GNBS8{|*^=!t!^#*NwUrYX}u(AI=EIvAv6!v8YK5tl2um_#SO z6btlv#3~)c={ZF;A#V+CkwP0DCe_2?ucPQ$gOutDRFeK4tJzM>0 zbFsXxeHC;`J7S`>EG%9ckj24HmlSYycJ8AMYvZj}GI(AQ=)o`DOjV&ln4UdbvG?vv zRDa~m1DbN1I*jeO>ki4UOy@6*VUdyD)DI`TIXL)`|8 z&x@^x(;;tGwY+ExV?(WQyVHd?wF- zoYZkoB(KjGB7HwlN}#=7ChYIG7TfB;j_S0IX(MRUovx(RYw?KLvmn_DGLBh4<$^DS zwmzR5NYU3Ei^g5<4WYg0QJRfSm^Sya#PGz3=(pF0{hGvAt;6wmtqL^fIwcm*f&*&M zuYGQb{1Nhb^c7DCvnsWk9f@>P(q;3OA7K}rQhV` zMWr-JM}H54?n|oFRQG*{qej-;zP;?&553T8;qC=6OWnUURORLZa#ob#i)YX}gR{D% zcXmPZX5c1#-s+lePv3EdGu{4cCayS1s}|ZA?q&}|1;Ox8x8{b^q+5X*zrH(o$M8tA zLgN~1i30TH=1$Tgid^c~5wp7u&ue%6)MAuDS{1oBLtE?FnQ;S&Ri(+!pR7dw-L}$O z*blT~HX6w2uy`j$3S37*#8&=FQ$|G8jMTi{ee_*N~X zKc;y^+8uxFt}I}_jzZxL*&!X6?hjsHMN^leU+QAfqJF=_$0}iG`aIE_=82wg9rPtr z$y<#e9qXk}7L}(NiW+dMcNksysF36$k@JBX826BB9g`uEc*H;GhfW>Ll}x*68nOc# zbw`y2b|qU^UsL&tHS|?G+cvn*zI;oMFK1DgQ_aQ`CLNp}%}x>h_eV3djs~}?Ax!8O z!S*LRIy;WBrzN=nfB;k_@kFuOjH_04(F7Mn4b6oG6jU}> z66F1qoLUdWUcs_%$#}s~thqUYoMKKgp--m|X?%y;dRb%{NQGRxdC_$)O-w_DWYBp2 z7S9Ax`7n-+NqQjpHBOQmBeLl!pi2C;NLe^(xH^04K!yJIREg+9;@7obhB=W<3e4Ao z>7k>YK8~o#T>M4RM#_E#3$M=&{y9&j!i3;$=vn=9lL8Z*=!N0c&ECB6gHEda0(}?Y z^LC)trER!S2D&15&@JVlN-|y=u&{iK9rY%(E={t`@-8moU9?MAhf6&-`&oz{`J=Y- zwomT3rLOn3r@UWe5g!U}y=i-SfIz#2_e4pxI>0Vw|Lu4a-oju(ApwxSQ-2P$v?9Ng z0PR(CTW-g2WpqrSd-xo%Gp9i(M6Lc(0tD_DbS{pV6V0gwU1S~TQqB4e94n(yv9C)5 z0(cAX&`mtV;%RUxRAPxO)BDKi%WB2MSkA%deUnBPrQB)|vA01b7*R6Qo!T`uK6@GX z=R4g~;T50y)3DU>_e2@R;k@dC{Q)B!%o1!&q>mBI@EMGqs(0#Wc^4n|6KnY#m|9MY zF}nO%J^oHp@HC0iFD7}!&G_IWC^gih`v_A9CSE|Xc*;laq*o~A7AAj5)}|_YopHOs z+c&OnXlOY&;O*q*go?MK7Ubgn-V?ekUUuGXY;MlH^`DKedALW{&@GRvrco?d|623l zgI$*84$e+;z@I%Vyq|Zyvr%hF>3-`&t?kU$%VTnz6uB#ZAACwR4FAKP8Q1RCPT9%y zl2OTv`1EZVzR%)Z>GpHoZ6%!iUhPF;Sj4lO0G9?-xhvhP7;vMKpF$&%WRM2`j2rQ*lK1aQ(`l?JI~XaY-ags$=>83 z@|S0|s|b=Nd0Ji9&UJ*!*K!Ynm=dMMxc7{FYE<64%dF72vd?d0oGmpDuis;{z7^Hh zpMt#3S!V4sME?=rM9tpNBv}csAb+WT61{iFxhsjHzdPqK8_O41qiIcDmmPa&dnt)? zXHHsbFuSSUl~-g=7_F^X)1l2P6r$)Oc|)0EE3~wnuRp;_OjN@OE4^_3RJ1IDt!^}^ zxDAvhPS?82mJ~ZoO+*lD@oIcbqt=Zv0$~2=(>z2Z2)QHy^f~w$2&5c)@0$|!3blLyO66Zgskb*!@jVp(2Up22T>Yb_`aiq zXxc%UOfet>Dt@;5qLgJJzF_jJ#@pPzuJZGH(0geOqc?=)K|?c2!4Fqz>1W_9Km)NCATIEwq~Y# z@ha}=n=rgSA*zBk1s|o<)MxcfoDZ*evR#rq_wSo zCENHO!TWVb+anmv^&54|Dc}!+UEgl9pveODADIJ6(_K_(8=mnXgLt6-gUfc-Z|K^- zDxH53Kxl2Np;mPAps~R{kL|@s;i6L<9$h2CHaxg zK?v5vG=*z2HOBdc#{xGY7u3W{7Y;4k-8aO5*=cc*M80Lk3aK z8ALxkiTkA)C{(vQ^Xcng1cg@GvLCti>PzuzTP2q9xfXU}nD7|~V&X8)ce4Q55t={IX&pMpUa zgjJ}b4EUO!mJ;L!GMi(f>if66=Q7Y6#}46PzeWml@$b2SIr?blXL^^H9KzU;VntAN zb+=9yAb&KnZUkx(aToom`+EY%pylItI zt-j&bZ=jj#Q*3Y-C`VmkmA+9^6Uq)}d#gAK=~ax5&ku-n0^HNJzOYt&D4F@P@d_@< zU{66;!ipsQA@_>$gPUY!pq1YaO%Pi1lMUJSkO<&UY1 z{tJsH;^r#o6$+(t4>Kxs)Yv;svPjqj zE}3;JyERIf^6PaTe)u0P`@_R$NpAdu+(FRp8~RTSQ$&hHm!1cV_yd5yDp8ykT(Xjq z(AJ#whWTa8ZfcfRg@x17U#0K`*wfx-94|(7cxwqdx_0%6812sbU-Xc1<1%39-U5v# zWzxU|Z{jhQTdDr}C;BrlGzwB_+gi&jXibZv?;bY?rdW8AK?WfnT-2z5uu4(X!V{8o zP@t@gqn+0|Bu*DD>CT4i1!wKMGGGYIxcU&mB+~*s7Ud_j9ZD87G~v63L~JGuFKnlB z{;aQljV*CbW2=*^_+mY}R&&m@b(t%EBOkhvZJ85IhedhJ>j6txEY)~S0Ym!#EK#&0 z@?g$?9IU#1UEyF-mClLxV4pCSz?VWQ4cC#jiViWkfic8|W=8)a-%_KSfjHeH(D^q) z@{YljY%{+}4PB?||I{zU@?+sA3>q&LH~YtZ^BSAn5wG>QvM)!oR+onV!f^ET>uzV3 zV+*WFsZ$*)Zk}&Bg(~IX7jho2ws<$TNZPoB2s(q$ZZX6g zgkXg4I*4hL)2Q#kkDPuCJk5a!s&r{S3t|$PwzWV(b5A}Ok{31OKER_|R%RB_bGM;z zd@PWs^4F|C7U3sNip5yfc^C1<5{Ef4mfMxreOy;Ikm;&DFeixAwjTWw?~RywIk5|D z%f$)<5>W&5peTw&2TYwE-u=q0Q9ZF=bNic1P2=6;BAW2l;4MCc58g{>1oPoDQBa_? zZ!MTf+xyp?c}>?;D_(T9Hb^4f28koEGdPuZtpZ8=w!SD9t36}V)Eepmc~*&)JL z`2V7%RO_XMa$mSk6$%EHctXF8sN9dT?cOqI8$XxM4^1Q*+6Pnx+xgrod0P6X8Z zP>WB}-Hum!IpQyuuc0K1x}sBkO$iPGGc)@je~@_hpHBLB$;=`Xh2bHj9Im6v#yUOv z&w}wh32lc-OVDNWWaY%LA0g$5s$EDClsl{7RFms6Inc*YrtcP{M~|?QZ5hwgZ*%&_t8esQ2DMN2y6;S)1PM#0F>)Xa>+E|H0SZbzCun&1IUAwynWgGp9Yfr3`vvZJSFw4jp9#iA+%%< zw8azy{zFg`y@EFu@7X_&BV_99e^$lQeWs?{WL$YE!IZ?b(7&2hu}-o>r>?ZF`08MQGc!pzicvq)2zj_0FpMPcpw9Q z$yI^{dtvk#%AK0C`^oNJex1acYVD8jg)u#;O07QlnveYuzkdwz0#W62rXV-E06jQ5 zzKCkO#~++pRMjNcq%5Ch?fNI~5QY-<>CzSsJgpp?wnF%2<>q3P?FO5UJ$hoxe~#X}bg zy_{FF%Gmh|pWd1L!88&2K`}&Of!5D$*!mXB?bSPOhE`1OoZ@Ow4QkydH$*E*t;at8 zzZ9da1esxWQTH)ZN!4@0a2EG9=GdGrKaaKfZ|S*?u#2XrIF=s+=W+0%`Rx3IdSl(G z9m-zZD44UDp?Oq43$gU0vKN7PPoECy4$(6A3d0|JM~`&LWrL%_gc1^jh!1$I+-*8S z>G9>`r>tS#z8!wsnns>GTgdtOi1xYW`I|IIq{0s5O=g3?GMQoJmMWo)e}2x);*Nnn z5X%;bljK|}igDq6b;$%or_=*cyRmT~j){Tha5ltWM|J)5IBvnmXa?du)P@z+tNk+p z^T2Fj5h8Fg9zzby8+@Vw`qNEAzi7kl&yyA_r~x<4xmy~Gl)gk;5RLepUPg3uHCnx$ zz<9}_+F#&ufMvTVjQz|nqmT`WPtfE2O6e;8a`?sDily#7Q2pR+HX4TX+%%X~czFp?CXdUX^<nx_PrOr(kutX~ioa%#KYj|L$5mXqjR2xvj7nx{q#%8-E3FE&XJ@ z#kl2ICp=w%#vwkTN0ZU&8UOqouPmcS=+HKdpx2*n;B6okmZ7q8tg&>{Y<6`{04KhQbsA_&HP?Gy_oyL1B^@h84PuEHgb~ZTy z0vz{yH8HVuTGqh$!Q2laDDi=kMgt$lvOTnDjebLISOs|IW;>Vx)#ezh1gQLtmQ3{v zG*l>TxH6Xg5V-m7x^e+0*y;f>liZ5wFDe)B0qv^PD2hDH$W+O%=~=I~)I@(=ka#j(=W=ly6^CY<)Piw>i8WpBjVSqZ2DSkf;T{5yBo) z8lH7B9}OSiBTka_neSAlN8qj(dnL`#A=AnMox10kt~UY}Zr$IO9D1&|R11PXx4oJi z{aL(^mMzxHtta04pOHE=8jPtEz;(cb&I8Q*`JCrn%5dy8JqqVO;{U=-WnuZN@JDc- z2(leM8dG;9#>eUJ(bY(G=q{^&n_UC!1XF%J~9YvEdH~MtM{p=HX1=WDAQkbPm zlDDE|fNY0tYO%vkNb77pxH>u^vCk?`7lu-@N($D+0*Z{ z`oveDe&RLIqn#$SJkh$`vJv95ZH+<4v$=2)&`B5@0xq*0M*^r74}ls^c6R4MH;@)Z z04s!y0$8A3rNDu;?d-g7VvH$y2CNTewqbo}*;^1tyg#$l62Hr60H3*_kJrMjjurEo z9J$M`Dc-+MvUUoOIF9W(`#Kq8eV}vZm?YEj>y#x#zcf1uQyV~5AB?zPfibg~L6foE z&DA~c3S@{9)T=%O(j?raWxSKcdZ3Kub1)pL%ZU#QShePnE9-`+yrGFns?Q-2+=Y7Ts@S(fjRQkt6jzPZ>a9G zy7X5Pzqo@`MtK^V=jx{Rnm@sIH3Gr!{NA(6Op@#Y2EXy@)^; zOH(gOni>CwK1BZEMD~KR%&k9x|2G%+zF3pK@sN^{!6v__^IXLo1WRM;k}B8HVaZ(8JgZvmq#MO?PTEo^m@^#F+q< z3*q4OOM5`xKs+Sahoj~)cdRDY&F)@*6Or@_mG}?npd*ph8skmDgJ!3rs|(!|JSyzN z8)F-BG-9+%lZ+S~{X+tLuCZkfyJCL5E1pZ_rYvU9D9+}YZhT%i_fPozo0Dg=Lv2eI zbQ}&I_HXT2@zM2+vAct_?vBo+Ic=Hq+J_gc7#5~yU*=3&qs>yD%yq5Cavu^7|HewO zeBFQ_+0|Zf$<_j0ceJi+t;JTuKVdLNL!Y*?uL9(tLNKHaPq7>KfUB35d+P8568z0w zXH5^YuuB!PC-g^Qw^Pte|K%q?|4h+kNis*l}gGyuCzE zs)u1mb{ds)c}H={cm5B$s4f+eC2fw=_;b+kdMJ)n5eFu@TP6lvI?I$o@%j?O<GQ?-b0oiTE0$a}0YQ&3_U`6sejy`*gPYW^q9j*^B%Yh7_&NQ>8zs6h&D0^wH38~- zXDb~m;j#1_n|r?OBfw0#&(?DmPRDw!CJ%GKBQPA7O^~Vq zuWEW3c623=3Xw`FC|ad;wUB%n)(d8F_yNz3GubE<;y zGVi_wOIWQ2Awv7M3@{G-rz^-{S88Qtqsqq)$9O&Cf{2~Uv`xTqA{!Stxa&qBCUTMHj>P)%9HMLTeu3p?6rOvaE7xZcOn%x zRMr&LBGHYb%?bzYszgrOOka#XXd7YKvQhwMDtOhECR|F8E4!IIXlX!69lpE9p_eBU@*x9L$O1i0|sF6=xs9s+rhEicnb|| zwqX2bno~O1!pAuikTq7a$n8rv_t|!cwQF@LU{q~JVfe}9&37Q2=y4k8;QTBL%(O4m z_;fp%#~tkh=O))HLuzr6IZFZRt8^AuIrLrHLZI8wWwBu^jI}xb#D)?$%xM6 zNzgUdb#bRDCQ&lFs8v)4w4UXFX4;Y2`sng#<0hJ)@V_0Rdxor~Vrlev-5V?ZoX%eX z1s}Q>-GX1OHH;;X)T&(+Jf%&MTM=gwAWgAd9Xz76@ce~DZ#B_4V-cuL3LR(H!a-?G z0M0Da-FB(SaWpYYS8BgKvHy?y^n2-Ck#zdiAQailcdtx%Bp9^d`SP|UIma?XQIgxV{r2naC)o#=GbF}x~aJwF&iQrN%DpZq=`zIh_?+8FupbZy* zi@K9NQ0|*|1hx6aj>edbk9w>?Rr3`ZAfw}nZ-GGhlimk_($B|okcHWy5-dcORET>6}^-MJE$u&R1uDybul@DB@s{xsFJ4F8*7GGHt9W(W?v^RJOmEJH- zS*b<#erGNO`zsYrC2KWawc#TbdeAB=0h& zCj6T)Qp>R!SL?a9p#Jo(P;;TTmh3+`<^N&tEB~VGy0(XwW@wZg>6Q?XF6mHW5QdVH zF6jS{ zwdn7&`my`V0Mf-v9hUI?9CMHbZD=qh#MOo_43@OtOz}DQ`wBV z-J}8ujA`L&sS(KDEgv@=v&FVrv3b#csi7u>5=J*egCZQ4-83B%-j$MpXe-~Ls9Wr#)TwbceJlwS=P?Y?Iws{8f+`soKLqiFj0D#4$r>OV zUqASaWQ6eP$0b`V9kN)9VEfVE#g>h3NE5lT%mKf$$~ksKb1D-ppq+`XOBCUs zK$m{U)yr5A>+^9Rz}vX0^-My8LI;r6N|+L!ATkyj9!&L?kMEAHcy?L($BKeTy8ToP#ossVnatDvhOWt+q^ z-e*KUpAoMOFbHi{w#82wwQ3ml0C@HHrR0CRBoUG!-?$%#+^G30V+miCTFb8R3QMw| zoQ~z@?X(;YL--%`TEXoADWPnYjztMPMQn)Ota1A)T(Zh=!ix&$mHwa}Ia*}{)Biqk zEwd%qGt@K>BLY84QOS91$^Ql(UbQ$;Dl7cUf$y#Nl0*r}Ih7*f1b^J|@Eb+8-2-PI zfMNKzmv5CmW}QY-Kv(R)6Slut*ubycpB{Y%ihD0YWum97{WS2cZL@+6>NkANzg!+4 z$M9`dIDAr=uuKMXi++N{0-oidQr`IF3k5nRLlw&RCj$XU!K*aaXjSbQ?a~NlQe$b} z{XQLNeX1;T-qTmONrwcg0wlFX(mU<-UVE|{0EjW;|3||bd3m5m(GkV2bWp%XBvRyg z`nH9^TXiZf-hsZAXts+*DhN;`ZyREOeLBSysp5quiSalj)$1};e@zLkjRwyAvi6-C zh(8Egq*TSz)a-1+d68tA^i7c|-ba{q_{Rpt!0KQFuf+2srxmU;s;=GWz38KfuxcyY z^5C{omUM4*zd?9uwhqv^2kes6e`pGakJQZJeZ)>R2@Pf|woa{YHqMI=WG8nIEfOw? zEKdw+DbbW!^i@dU6i~;=O*vc5?mlss$J9oNscF9mS2E3M1nPr7bu7mrbBOfkH}?XW zfWGAD_jyqq6uzSTF;>FiOK0a3$_;3yt>ViCvT7#xvs48ugpxpCkM#=stA7Hf2wQ|9 z4Ap#74+_jXWa7qcPbg@{=PmQoHU>b{7OWB#t!X@`dB9Omxam_bLtu0D#CMRbWzJ)q zC9i2ud=o`ob6e2jGoxXYcov^`D!X4AII=sYF^oWAPawGT+`qN)P4@B-FMwLRDbpIL zwQuW~;f1AG06{_~wl%EOA#fD;2bb;kWamv&$C**F`U4kD_w;3tzdiKF;uwjlby=%rKwkE%e~N>) zhmFx7`@(tp4IdapJ&NI(l-7&>`{e)y$$VaW>rK0XlyEC+yGR)X$5UQ_ELN%pY|&-d z9yFN!+=Dy_5@Q@>$XS5uq%6>Z`6U*?#*At@w&^9*%rH}>IR!E{+7Mh~P7}eYA^V#C z=nMY;YLZv%^GMx)lkmn%i|66{b;+IjZ)8205LdW(hJ|2mmd7LFzRL?bJ3mZ46Tx%u zXiR&MB!jz%&QLO~qtRy~t*n9n*SCK($(IQqR;VMC-_pwlPQND|40PAtaNP558oiwl zr;}sXgf@4-RZD^S9SBL;#hg%=xw)TFoiIYxG6oY(Wg z1;wawYE>D0Bm~z*>s^z<7yNIaBjWTMqleANmqhG-VVJ5|1ww}A&nO}|%5^lg^YNqG z-E(#N94nuSeO)V%b~2b?aw>%DfJ=DL}(dUShS!dnM$6T1T59T7-{gf zpxRre*ink%HL{wT$!vstqHiV7Z3;rT++nzxApYR>Qk?7?kpFO;bo8;959ywjeU8=q zDBC+!?a?KsFn}ORGQy+0yHl_%*a+XIzf<*6WVN@=YQoYbpR>-nq^N6l8<8yzN=rrb zjdYGP09n~g&{+^FTKdq3_>WfezJn)=45v|;??+B1^tTH}vi8?;PfXrd?T!xg!Q52D zc$0ZcYCB#Au9{(_<`m5+XWU*WspIsjU!)k(2yo_LPDx?M!6vLFmly$N{uqKr2 zGL}4l*C7`k?^>ON=}iO*h~SV4HMXVThV_S$EFmmu9Du_7%TrTA|9uuQ(u3zjw<_ze z!Kezq%p=jX)2q*&$X429?LhO0~^J$45DLgS?i$AGHgl8X;c~8 z%Vv^84A#LdJyv+sYlKL6^=6emDb|VMPQ-3PWpzMzU%%nfMoDmnNRzYXCf$ln96%qB zb?V%F-T74{5Ae88sOdo3)P^^B?eU+GrXc^27IGl#oh!nYxw=6}dl&rnRCbDP)G=fI z4ckCznx}o%D+j> zu*tR9wTy!!k|xne@&vNxnJ&6;iGIj{?xFfE223GuTj|jE0-#fmn-}B_4f7q<5=J0oDIb`H;)}SpUe&gbV<}Hkz6XAfu#y*tz&lw1gBea= z&G28f^mC8BNMx%0EV0Ub0D~m*?}!mg7!}{83~5Nrex0Srpy^$AsQvfY<@&V!v1ad; z&z&it&RTq+vM-G;4OlsiKH`(AhO*zHEK#h{l;hBMZM=3)F-DY{a7xAfcEMZfix(9{ zvC-wUM)4G9I+h3JX5>h3U)LC77^We%@tyobwg}n}F`Il)jt{o2hXHFWltWh$fR(-j zbT$&@D}74u1P?57e*CIZ>g|?3KzjuVvJVWY9&rDRsImt5>CPU!4}aYzexIc}`INdv z`VqfBczt$|sPMU?d1PI70p-On)h9#y8B|bO?E?17&=BfSwXKb~4fVsiTN~tf^;A`u z2s7E#(N8QiOa)?c(quA8YvYmUHY>^`Zyt~kg@+>No(G$t{B6ZA!W@IMnSr>Lc1ml{ zu3vP#uAk5xDwo8}-kG!FTeo#}!5-sLxeXmC-Vo$8&DZOPw*@}>E_)L+BlG8_@o=Ym zhjfQJveCm&f5uy}_9i93A8pKTqIRcWC@4`(#Z3sdbZ=Qn`Fd$Ts5Dlq75#*6Q<+6L z&oYGDLLZLWU{seV3n|zLEqXL-;8fk-CId-4Vrl2AiDy~AMLm?!}pIq9EQi>)+IE! zn%*Ho-M&9D4XBTySal@BVFe!J2xd6B^~!b(H=Nd#%MjElkTV`PFp^q=4}1Nw0B|zI zLQ2?71CvRsN5?_L+!hY#o(9&wIPEOswrC))DC2~v(jF2hKWKfv4Z`Sg*>Chfx zsAQV1Ao5!;koXRfw|_im+BP3I9uss(>*@V)kz&B$m!EVwUkr^;t1C5mYvi#*Q3(;b zRGqP|98RNH2Y+khepeMBZ;~Ei#jQ!Na`F-!#nGf&!~$?a^pPyI&kcp@h3dnxbD6Ig zz0Fk8&ItGDmOs$FjENHfN;w076M9!YSygMubJ_`*jXy9DclVt>pYFkJG(=+ceLDag zv#QgfxB4vp$+zGqoNIWuu}yet3_Z!ukK8E`_eOo}qo79;`UMUj3Zx8P8JREp^H&Dr zjpCuuOX_5>8%tT{ma;g?NgvL$x3HIW+FO}WZ9fTrtStQl-*eNYY*C2cH9lv-j2QvI%1K`M#xFyW5)HaDUytrlrV znOd{ox7^2EwPHzpbQaTf_TlV<=PNwmS)c@*r6hpBFN)-_>iTaMqq}{(sAv!t()?9U znJ(Vw3!)iq#Ndw610n7Aa+%|C&@QAcnJmzvDs>mH>U&~=9*QHFk#AnZWLf$ddvIA% zO!6Wcosw~Pa8@2VTR+G#c3F0Wx$z>RCG_W-{0{^B;kKl(RRK6aP zaY@yso+NvY_2Q*SyT7_gv5Q|&j*ofY-`LV+OE1b^ntVdvq{6QwMWA_;9AY!C9APlA z_mp(c)e9rP$c}j@aPI3xpTGY6Vx#FiW|u)0%Cy!r!zPNBIX@M_vDenuzZhHPyHF$N z>gy|Vq(q-l1BITf>&<)~ZxO=#V`B}5MIAKz3SPH!-=$wdfSmGvka?6hFz0(Jd%Hq(5q6QY#Pw9Ws z|3nI|8eUUhQ#S*$ws$Ma*O(GI*MqsY-A`B8HQK;HGV!4NtCIJx3gGL>BLD~f$1l>r zC8q3AxJMxsakq3ca>g8v_w4&}J;X<**izyE%MNQ=%{_?#XcOlt`3rdTtbgm6<}aLj zvHmyg@|=XZOBb-0SpG@QQ3PN~ksNA7e^&bTyab$2t1ZadGrEezP{e$w&QbBao>y@M zPHyJhq9m`sM+Fp`_eV2qaAuUOTuWMPUf;$2hu8IdX7>GeS^(o`Bz}pnNk|^WVYTTxhZ9IsxaN zNtaI@mDZmdS{o!(Da)DRBcp=O>i{veN@y>9?GHR_e8(oQhxQBr;^hHzVnh9pI^yeU zC%}ekg{MhhO2ws(FCXRzd<=~wT4djeZ{X^jdo&zJjsLk-;wc(=$VlWnYm8=I(qf2H z9nOMj3M(d6?rh^GUijrKv3u80@GUOii>@LG0f1eDiew;0_#Cb!{npBc3i7$(b02PG zoZax~eOfFq+A}28KcIOI36(T4_>)jQS~3_eExvj=O#)hk(6Ixmn+mk7;b#?ClncJ+ z!crm==a#Ot8nBixOmOC6tfWQXHUBXZ=ze#|^eez2<8}9%f?isC-+FHp8`|UWFyF_T zxGx4d#re!dK^4ET*YLPqX*S)#@vaCqHL`P;Kv!o8~5|SPIGZ_&vE6OADC`F!?=``r9(KdR7XSs zEVi)?=$&PR_40MWzo5RB=yq5IR5S%Vl=VcAsN-A=z?Z@9yFPmGW!|aYnZPGwaZ`0W z{Hc4@m)ZC9@Hzkuv^qV!tNoY-kR1(l_ftB&*NGM^?HRX4jDc36&e-ilKIX=tMU57H z)OeP!<4kp|CCj@u?0wZv!q-@!O`*(fRJ^vPCYn6zCw?awz)Hyj@zx9|bWRl}*cuTo zZA~L=aNy8>x_PwoTF;AbD}IY!fPnDwMIQJ)4t6G;3$?I3yEy1u$sW62gJO8~S0*_3 z3BQ%;sN+WJMk=qV_m{+PCMv_B_W}|I#l*=j#&G!J#cl25kqni%D1%-R}xgR#~&FFjqidWg( zV4qkHV!I%+)MG5Zu!@FcV76dJRZ1zTrFRH~MuSL9e z!wmZjAzm7IJDLe{+@CTaaiZq%7>(}nAln6a3{ya+68CZLBf;%Ur;be~kj%V}liR|Z z9cbkU&sNqJqd*QZpHLqA`;y0JN=PO854L|XaDdzTRc8}}`xoKQ^wn$b4dB*Ui?e3| z1SAWpVi~|OYP%+McFPr`7S>IDwJ0X2>DLlqAe#Arwm3iizg8*l-96u8rqw%;3$>#n z7PUjXu8=8nr%?Y@Hw^rCuIGTQQhb8+Cz=dZ-mp&CRst05RcHR)QdYW`}{in$dq^GWx|;2PC!{ zg9Qqsp3uj@@^S;w8=s-HS0Dk01W*iBSUn2ySH5ac>kHov3aPtVb~n+F(WT2k6VNwW zPCz7Bn%9*05e);+zVqWikU#>oV}@FHUs=NXII#OvRu@A~s;IWW>ccKdol~X};H6HM zf^cDm$AoI)!~{gBwubHXJJdIegX6M?7Tt%V#BUq@>ijHBOHG0ExsC^!F+F^f;0itZ)PlDir{6UKC z2oM)hRM57=1O$Hiw0niGq%c{U|8AmCIKqH*W#aG6^BLaz~LM1 z;UC3&)Se2R>mmH%arG}jF_RqfuST*3LGR6G~_We`e zI@9^mfZ=_G#*+X-%@@R6_JEb_JAfYxtNy25%)sC_3-fTR&XyzYZzyhE^BXE>!a)Qv zXtEX1WD^6a`cn`2*M}cP!8v4zL4}fs-isF)ijjT}GWuIQMQL)_hl~cb3btr(CH)Jo zgQ5DXlwSaoeq5(JpHYE85C0cL*sMZ+2lu@|cLt(_;$^XG18oGb$LsK^+CohYePV2k zLGv#)?YS!Ze(}|y1q9{(qgg^4hqOCLzF=i~(0df>8F;Wf4}5HS_lfX;>6G>l*bt1C z+}F2y2LFlsIzb8IbpTb?Q#y<;kyV?hvkXIJ(NaJ+^!0@2zchvhZ3cq)pV1R#i7f@8 ztA+!^M*LDQw{DI^R!uweW==9(q?4Ee^J;VBsO=dItQ9w*N zN2Q=&i$}fy|BzwiSaRU{8}L+GvEgn1@OZ-y)bVjI2gKUr>lk*V;f!%}^BH(plwc)2 zk+{)%@4W6mh3i3flhELF($hSiYCiSsS^0i;=C2aF-i?fkw96!{#4}EL}*8Wnj3N5kIPt(qPa#?rl-E_7in0z0N6NuE;SLq z$VW3-i?gta3kFeb+CO#K_<0kCo!i-pcU{HT*As9C2ym>%BsY=i+olf}*#uAwJ)#%w zsKPR_U)4GSQMI@dq9!-2Et0oy)+kh6y)51j-7t__QI_x~q>=y$2MjfXO{P-k$3g3Q zk(ujv-{Ul`V6nQ3k|nX;m6|XwF?V;G^SnGw*q#|fJ}1)A)DYKLpiyFUj4H~)r{-Bq zD6x3+p&lwdd!%_@yTyvK8jQ&a(44XL&rQ!!G1rWA>SXy9`dKc=l$Br5t4`W23Qt60>&NwTS=cB^svESzK*uc<4hqj#~1b$X!S!&*hv@tyw?)4e1eT-witF>Ol7fh@2d)9 z1A0%gygdUtpIVJ~r8FhnM<4*;8mp3-gB!2iC*7I9J&8CJgTI+0lNV8@{ zLHK~-H`Y>cZZS@K#L|2ViWlc;A!0eBFy9jr}O zbBCnxC!MGj|N2`H?-q0#bn-r5D8JC4%5Hn~9cKEs@?Zw}zpwyd=Tok>6H&WuGg?zx z6q0QgOBPEr70FUSB@UGif4*9h>Yd1@k84|3*Od2ge_aZFW%*wTGGQZy0#<07dT&uC%zFe; z4`B{}@Ojs?&B+4aX17eBC|*ciXYynm{2~1N>5NU21WF+@SOf~|+K&TK2Yg#m7JJ`= zec=T#sNe8^0H9UaPmQ9uYjGYO=Tm-M`gN1L(uu)JKAtVdQMz9Uf2Z(3>UR_rbuTVZ zDeIy>dXY>?KlX^KRQFki2LoUaNQv=Bp*!F~9o^a7DQ347Fl#ksCB_IJ)BQ07pgh{T z#s>k#|EhV!Q2u+1EjAGO0Orbo$=SR*N9G|_Ctbh3ox9Y);9(Y0h`uKF01Kr_+L~Du z$=OaKn-IbAhE;%e0C5~1b6E!$2{Dll4=o5c`fyvzm-BJ)RBvVb%LMzU650jHr8eJ> z4@(57EDfpAk#V)Y7?m5P2r1AoQ3qcjgW=O~kZ0S*IKGrWvnZ;(P}Y2J2o+QoWoS`= z3zgF7xTiHxY+`A{_3#&qJQ>v3HnNl(D21$Qa=rhYf6aWb7`=?VjOXKS%Q@xOqssFA z@mENom4Z(+kMDgS&#e+N7xbRQK9I2CCw`QJaMMZT7EqvB+;lr0+B>_(h+s0mSFd5# z|6bHHQ#4&fR)>=TU?FeByOIS?Pw$xavK|nOO>CL}(9QR>)U5ZPoKG9FIKM7cvX$HY zB#v0j6dcaNe`fr_IMSC&rs-mU;DTyRb6VkQDXKFC4U3XbkL`y3>nD4ceP+G*_(Ox; z$5Uy!uh{&(0Qpin!UpKitI2Qo&js2L)hICbfmd!e2&w*^8l?AhB?HaVfgcZr@>tgV zQQ(1j1$f50g>VU|9%k-=0Siu_zh1Jp9|q2gqpW}IXTu?6dD!D}0DUQ>RfWI07!9uo ziW6nH|IL2k;Sb&MsI2mH$zY=eG|Kga$T0O(UL6=yw2&8mVE(uHPZQrFU9OU*HMuI$Z7na4mRkt>IokPBXD-1!^gLk?D&$W&dT zwqxG}LaX`^hUjq!h|jtkCxS~;Zu=bvIjZ_@VF&pv-o^dv3pe$$x#7J)|drtuBzpt#YD`C z(`v?Hhy%mnfmsS^fX1PV_KgAn7|d7N0@{<|wGMw z;>LZA9uU6rvzTD9W=Ez<0G)W`>-99z%p$SsZtvls_lSfByy*~$Z zL3ZTf#B)l^m!)lh!882q0FCmy_jy|2vBHenkJXOpfKlKj=FKZo`ET7Q1USRnaRAQn zOSmI#5Y9yZ%picLor|(M)y>*cX?M!@IgCT|((Iq;+P!x$AU*+IPm5B}V zR5G#qd0xE-PSkg`mCUJ;`k9B7Dqw=M16*JnkiBUDP^8V)YTN+B?S0OFA(7QHR#6Kt z3w~;cG+l{1#!iji$5wX)ZwHpW5*%d!rRF+HK*&f8!N?OGSc8dXFeOWv>mvu_y74{^ zj1x)K3+_6M-2mHP`)^mJSZA_72aUm%%{Ak7{!c1sJ@975E@d+=tvC-D8T$6%CYl2QUSlW8eyd|= zr_WYCYzBC`2?7&+J>TB6eVd&3tdaOU_zC#$X_=Gd?r<1_Ib}&;?Wd_vg-@Q0BvWTWPbPqM!+g-OCHRn`;tA{^c+5$1MTi8 z|B7+q0(3K<8khoG$_{@y3lyIp1K!h=v;a}7_A0fXA2jQ3=#QJHW;1RG3c34sO%`}S zU-1r5ih1M#jXOWR2qVQo=a}pFsE5w6PoD*7LXaTpsmi9`Zl@`i$1L&w zaULlZ(ygpLtZjk5{opU}(Ur|MM1)|0F-(y@dCkQGP#5VqO*s3kbo$c8%x}z3tm5x> zQr(O#gpnH52FdjCXk-CQ;5A`|7_;X{W72Bs1Hfaib4WYBBiZMeDlc}f;0B{_E|_s$ zFrFmEPx%c%V31IAVt9=@>rX$b3i5pgM_s2_`EArM* zbHR7AKvM-rXNu0O&LM&EqX1rw8t$QKg4vN#G#>AEu2+@3o9$-h!LhZ57#=0z=aMH6 zkB(EG9}+*m;=0CPYCd&DSuwzhZ&cXav3R*^yz<;%tY848uUw z#Vyr@M;Gv51s!L>#s+WR>CcNF@73)VqF#AD(bpi1#ea-O;~2(;!-iG%1!FS^6;4`= zUv~e#G{e{ztx0A=SHAAEBi_~liKYoi$<;O0 z+mReJ*+rCq+FVS>yxC)VXPZIz`+c?U;tfIftG6-BtVq8(`oYuFNO_2AR&Z8zAxV9| zv_s0@qjbNe&PA2nc?7{H?giosw5Uehv?lrm(xd!fi&qoXBDeHKueN{15VjxtOkjy% zwx>zXFJDRMo?m~#E~1ybgD>ByA!#xfSWliH1K3=>bvcovWlsH#6p=1(1=qahbDV`} zU9O6_np>X4P2T;Tn%t2ZGTH6G4X7{n$@Wxg#%Smnh~2LVo(rDk+BZVKZokoea)*BT z9WVd$lZh6Y8j=o%%~b2*y=#+VZ%0RWi^}|YL%(~Hc>Xn`vs+xdt}z3pc%B#05Ih%7jP*2 z6d$`XX!T8lRzr9{F)$G?<*2+|V@O!9l787~H_eIn%w2OOIUL~W$x0lu;8f!qR^m`N7d6qbRee z!{D@HU&Zd@tr36EpWtUhRM0rF<5!xxC8Dl;>))=t?0g;4R`@La+0gU56N0^EP2<&;nG!|fj- zyZRO#-gqdiT7_A)3+$>ca^Nzo-gOZ`LzbuGX=i6*(Hr!RUd?^q&*|n0>yCF{*WOYY zTw0jPw{n;d)_>YqP-jHw^4+ZZty4!M3uRJF$I`?UiR=({aF+JTil|`T%UP&OEso}n zyPa#UckIJY-jsd&i>{z06O^WL-x9#BIrCXRTYL8y6X|@+(tx4(m2fhuQp#$jZ$Nn& ziwayv=z9J$*athHGd0-nJC*;D7Oe^_cCs_YJ$zU6@}CUgXn`{bLiAYdAz>h2Yt1n_ znmv+x?)DFC8&3dgDqt6bHNZFh_|R`d6_zgGTyRGPfG>W+*KO3M1%$up?@cf`K)eZ& zB_q+#opnC2gXWeneVc~GM#V_FC)Ed32Rbt(I_?T@)gEZ4aRBEsx^~ie{oe=9A5s8^ z$}YmG*z5AXR1aS{&@M4zrV+!WBA}uO0ft)nu3cT>-NrP86*>V7_W8!jQp+uniGyYScX147Ii%7a?)JuA43MgvaoD_TzQVoEM%E~Xh7Lyhu1 zP;i3-$)vaBiO76Vx}s>lbC46sV{k@b>>6tB1`=iDkxEN7CjYV}re}!H*=?m8ay<#~ zK!(-K`*@sR`gv6`HF+Fli*632^IOT4?AsjM-S*4p!?RedM1TDj6dIC>ga>$cwFtsT5?M#EHIjGJh%Om=LqpG5Fd69J!@^85<=6(aE); z*17ipYZwFCG8-(~^H0bUa78rr{hm~HJ{Z#Tc$X%=QZIiyK#gExk$&aP=G~C^Oi<6Q z{~8t8I5I!y`+0b17$p5kVzm}wl(tX3lKQP5WZ&bs8`1WD$vJ@B%h9}0@fdB^>YzcD zvEJh>D=QK*&MbIY;GYq3z0v&4i{QoLDoq@n@4Y|5--brvmdYBZd z)&uyjgCHb9rKFebI!cVz)@L+5xfE3A?Gw4=jq)pJOgdA?Q`lY8)taj|a0!|;D~rUIDz4|P?8(6&vb-=2``H&b-!AR``Yap z7*tB%<4SpF8f>mq7a@SB*)+hPigQOFZSftsRzV!bf=5B6D2&B}Dk{Z*4TlcDg29fAh(U5bzjxr2%h_x0i6bndd|l5qMEo)A3d9lq#18*a;BY|^)zs-B? z?sH=)o7?Ow4P*R2TCw2M&P5l4gXI5mz}QV zEYfgrLSfF?z854Xc0YGsklJf*5Vk5O=HX5IcP+wm+bhh|JJQn<@m0J6_%%(lO)R2^ zh)lWnuI1n(*yh3?s;3Y$v21Y~c`+-W#-^Sq;NMs^`G{9hZlcoI}9dMX=*dqd^<|J0Ns1OjQuGRT4$jgWbyLWDtEfL80 z7o%8F5k9@a;>9bXt0hSi4Q6f2_xw)s%3*J^{L;!&Y9UGC%woxh@yN=JqwCdk74?;= zEK`vV#>Z69tw%8NB09dCH#(LuuuK#InqDXo@HteEU^K~*@MDP5PIWjn9T(pa+5Pa( z6ELE2BeZ>d@Da?kZ3HnKqhYZWl}ygDy3Gu%&qvy3BRO+q-j+~ggjH|#%cVXh?gWJL zBiQ%|%Ly73e7TGzqa5bZJ&yW@yI-D}XJ8mHL(-;D1^gWuqYPvYp9tZtFZtHf>)%h& zK-WNF06qpF(pR@DchI2n3HkfeW~1bhF`0F4Ge3VroQIh$$%zn+WI-JBED1pzuickX z3Sj=-1ep6w7w80!n(M~v?@wbc@=(#o) zu!0kG5}7ERf7c6Jl8;ftMGle((_giDH9^y(5E0`ec$bMGE5DM;q%(W`6WGK$U?nDs zR*8Ktbnr;y6NB1y))FH`<>a1IE9ucg_||r}WaT=m%`bECdWZXnIaW&;_~0|!Nup&l zwy*3y=4&*efX^9MPbE^QprP@^9U$_5R?|^x0_xzT238sv@LsyOhOE9_g>q#v5TDz;-A~Qb zcmz|ETPlf?pg9jC42TLsea#o|MfSU6Tvg6|@4Y!pC(}J70+BXXZ55dn?@Z>w&Ii}u z0zPq1R4*$#SuJyyEv29^ViR%0=_+4nlzyoDGZWJBWtFg#F21ZYL4$pi_H+=@ zSdjpPMcne_Ls(#<-Vn117Q`u03Mswwmszsm13q+T)S2Xe0;N>=niibk=My(b0#3iXP1OTC9~gV>RsxBn1q~Hy zwViYs#9;|b7)h`twW83l*Kke1=jUq^^Up#4{WBIB-mTZ6`z<Cpp%1Z(Enc>=s~{I$?KsrRu}79rPvGeDr=k#ITm8{PQ7>L{>Ejqt zdz=HX(W6B`Hn9lpv6BV!0NDg_wiFA2Xc!6#Y01r9w@K7N)$w40ov(i=U#$UYkv7NV zHRO&0VZ*>BXI^0!j}J*QR{oaml#1k-ckkKjOUi0yM2FX0yDEhty2OvH{IYVL$Dic( z6$FP8wf(^3d`NPk1nI0p{+i4UZ{-t5s=^D@NU5OQ{)52{5NFpUcKZe?7Ypti`a#6w zkoIx<+jvCNzY9l@HXa5OijJ_mI9uWcu2I>-MMg*tli%4u8VMRS4dUch2kdU)Z`yQc zdF7Wm3C^`Lx_!zCy<6ChOwCMuW$rf<7v+iBo{s9HS*6DwWOg4`cJH@3RLg%R$t@`H;6(d)KH3#SM?!kW3`K5V42O(jy9p@zAtC>W zPH|2Nj43soCM;}W;yatY$R&>LViXwghe&z(<^3RvOJ~lRtpmvduh(uqn~lzhG;duM$rjF|mZ*GOAbf%3qv#xHgB zAzsiY!Peyo=5ufTXDkz(-M(ud_I2iSQREUKuF!%?y|3QSf$03fXc+kj&*RyQhzLXu z7(89ca3lb1s7|3V4^S7`pZoH#Sip=Z4g5neFR0a_;&h8?tSn|yE zsu#S=vI(X>s~(*P@w=S{uKNQGT75vAvyfUTSdKgGwq{&Cl1KuQ9?|UoE}m@Rni5>G zlOcVrgg!Aj`&Ms;1!5N770|dukbu}BCUo3#ky+a2Lx&TeZXqW9E@bgw2`gb(jBVwS z5fQeZB9LL%B+nxw@RYJApmK96IaAdm{RhBq4I&Xyzgaw*qoHqr^hdz4s$8k}yd5Zs)&i79o_&+;4LyN$PMEG;w&*QiB>4j>?*Q<%{R z5@4GMZp5q3mO$ z`F*K^aXI2{axD%!m&KQxZ#VZs?I@E`^yn&U*3}{HojL4Owp)h=_)5#-!nz+tE)SK;GhVcR zj^dDq-d&Ohj5ZWOU{|2Rv4DmR)eVo|u#Xd9L9U)Le zY0DMLi7`Q{8l+`1fP$rKbO;Q5W6sMZWq!Cr`|BTeeteQnUDy_{nGr@VKsO}G(Hz?o zi?2uzSz-m|I8v%QVSlrmm%U26DXuC*T$fbPlz8V_gZt}r6>Xq;fUzq+a;6se^pO3W z#gxjcja4(*j=gWocgTk*B1N;NEcaQbY231@e>ECV9l`^g~04Bb@>=^!z#Z1En-Jy zQg70SK)5}3O(C6i#fEd8(Lk;>9yaL)fzQIS4Jjg#mvBt@Q%EAqo5a<;6a6lo;nVO4 z+l*$e#Ym(moclO3V#jcjcJdnX8akBpoWPDqvTp}&2aZh1IS!84O;W4U?sUuEk(l&> zj7UF5GPBUu+ZL*lI*+G*hA%^^loQvy!IMdRzfM~t@|(K*^syEKBpcb0z^rq3Xv~fg zcv5$gxB+`|*$+HP2>%Y>juYD_hY!oa5?sOuh@I6E7FNK{_U^R_qH-{Cn3d?4d)|*e zjm^vYJS+NJjqxd(xxjEvR3?j&1%yfd?shj#bI|2++n^SH}A>r=TkvB>1M zyYu6pHL~Z98brO1cXBeCULLQhgRN8P5E(J;!&#dyWawF-arnJf~gLymz`IfdtTPXOfe-MS)8?(vB*{3_3mU;8#As>mc^zAM0i5xj75$S zZ6C!*zZ){HdY(1BS&T9_iobRpl1qr8s0y?6=UACeuSXg#B=pdbeOF^V*~;p#mZa$Q z6?e28`T*HVPbak8Cp+>@@kpPI-1Je_u1A(b((5;~a=lUh;(OcM6pX{xtHraWYoZgJ%0s9 zWOVoNX-f5_J_4+24fY#Ux9GH*$Dng1RjS2Cy=0eb_L8%nk{=fyNr zpUHd|xWf+(RUAUp5Kwi|V(-x~jVzjHSs{iCfdT`m2R)B?4uAcrX#r(mGh# z%!U&0>azH5nzoW=w2zLB`U5lF81FRmx-dWFByjkLqzAh0EtT|Va|$Ax~x zgMfJ67qG|iu*a>?PyDEQ9PRAjFx3Oesyn`hJ%%5!xen~{RMmvjV2`h(Jtl+%TL&L* z0SbfGFaY=WRgJ~~P(BWm(*j2-2Q$X+9#Y*f$!aIawDE*jyRf(tfr2aGxOhh4g|rZF z7jTFo5SZA7-#Y=0QSwk`U=>CsIKa(uU!4Pow!;_vV6H-v~MJCg!?Nr;pt$NX`u*a~wO%MpIiHYo(J43`C zBY>LLUG)IQIh&C|wsHyVaU$&TZq#BcVG$ESyraZxWEor-N@{4@5B=g0^Bap9?qYI{_Q6G zTBPV36z94Rj0sDNv|DPf53&n&_$StcwoC~zXw#SAwx$NmvMVI$^W^3c1D22 zG4>eOsTKOIUL91sFH&uT47u94k0QSC@jZ8jm_7d9+a6nr_`6(=iR=vYhdbKC9$%w6 z6aMg0WPm0}nAl^qvl#UI*P)+lgp4-#puk?|2rMm5!gcu;6MpI2>AE-?>pr<$r8oSj zbKwWO`v&%KTxa5Yr^DaAPj&t3U^$^-Thy=MjGGrx`{ji%!}~sq!$9I{IE?87tXD!U z;|A@($T+BtneMN#=%=OIAb zqFjK!7#R@vBGUQ6oqqZrYY875Nvp<9FVqqmj-|YvF_Go^%lQ9XW5DKTa#1^L@Mjpf zG_a>#JO+F4`ozF~?|Mu$_WT6{=#hcx_iLelzkk2#eaO_Q^PlnmR^~&1OHV9osM-%p zO^<|H35b)}Ip6=OX@9`W#Air-@)#X3ZX9)(X!fsAS2IR#f?LD4D2*mcoQssY|1(BX)$iT>Lg@9cu)gL${}!-6JqDVM+$Z$JbxVmSA&p(-v$GQC^&HPViDU516M_I zpv$9}HC9&2xwtNO!N5+%`#;?a$GV~6C>t3Rq?niE0RBJ*gRBhD_gXb#zVXu+SnUFV ztI8;@{t%W(4?}~ig(ats# zx1!l+lqo)zGB-d5&~*qb-!c#OxEGUggdUn?Qu+oDiX8u^-fG6{eS@1rzp<1 zzITY&W3&%9`hi)fQFt3No?=e5=dfzP@9c)aLw*2@B^I285x{MS)WtTpIpgq_!rxg} zF<4J**nt7KQH*f?DXQ~(8E?-Cud`DJH&&;{#tY?w{g)wd@d<^&NGz20Z#}bEH7yEFOIZ!DcR=>A5z~77OTV^#F%yxx#QU89` zx3Tnh6b4+i`g!*UY(K`h_NYzYyRYE>pw#`p)y~nMT!E#d-=H1zLB^YFgpdB7{k(td zF#=bo;JQDAz`+dEBz+V9$~joFS?=oa^w;gS$Bhu!eFP;FrlI)!Q#0UCP4QumgJlB1 z_`SFtkg>!)H#!kbu%(FQ4quCsj%pu(kpaLD~?I%ub2 z$=4Y4Ta$6$b{^@&5iQSC8dezXr3ErfKS74|?bsDA5=(;ZS_(!2M2vU;AJJ9y4zz=h zumtV2Ha`BJ{e9U|(ffKtB0$g&#hIpJDd2b%5pVk`?D6}w$4<1zO>wXl`)Y;{(`y=g%aY4!{U3wo6fM^-Y_iD56Uf(>rONYx(5 zjQbY-*F7kb8I5A6`GMF9JeA?Z*;w=PDN^_^-E6$U8c;EC6`EMr6`4IzSRy}kyy_e1 zH)`J-B6crW>@k+^))@?c3`<(96~tXp+9M*J_Bg2QF`n;+%=K@+RlOhAGvJrMH!=-}VH@fFRt+I5aF-e6Mf-ge z4?PbCU=n(>uh{1J!ydA%Wol*b|gaYm9RwC3S=^n6=E0}vn?>d zeHTSjFZs?#f6rZ+LlesmAhXVeK-HZHY`=tE6sui?_Z>IvAn~g`ZjVgoudoZ=69aHP zHq#!5ggr)psveeTzp+kr6qasRUTfInAgRUkcRkfjb!|ET<38$z@mi&4*aRIl>_Y$6 z7yVeQ&Ex4T>|Qy2m1up1Oo-~fLH4C;=GVG`gl#{t-`wL2pFk#&OQ$WK!13Ym3x zjWGT#ui$nN^cXXin=qgoiU7`*?#BBwf;?~)Rq%E1i<)L%W7m!D_Z$D08(a<)@j;BY z;~e|4>XVo$9c4LsZ4AdbhNmv5spNehH$rOsd$41d?Dt{Exzr*w@ewc?gm&;Mu79O& zmfbVj5h)uY3%(A$F+u$Gl^B3Sr5WvUNMRIjFAvZ1aV#+$josfXzlZmw8UN={5`I-( zFxZD58Q}4ITLk7$Pbz#|O9D&7=BnN1*>nI#yyCwfR2Mf76$2mkV8Cu%=xuMS&Kwus z9YV+7RO_vC`jHw-9F6OAF=|}AfFiB2!wknY)Kbx{--Q{m-6)3oFm_k^XONHkAXsq~ z>_A+N)b)>_gn|Fo$Nw_}BQzanWMoBR_o|_Ake_-9x5rTL_i;%|kYNQLkvUVXtLm83 zF;Jdkyx&d7J}fW-CVj9Ze9RT9)%O|$W@cRZ#~vFss;)#p>SJ7wR*h(n!^$2b6EXs| zMIVS)9gaX&G-{CMFD-OhJ21x0v$5ngqm#YIdkQ9`yUZ&*fI<9>as4mGE|JYIv$-Dz zao8LuA?V{j3}6+C2N^n3k!qJRFFemfGZ>!pc3ICLP~_BsUU_c^wn1gp(b%n{k+oDT zCqy0BOz+=y2&vkS!I7N(q}dOINZ`sa)cuhf`wnIt>gN>fP?InTa9Dw?MXHZrrt+4z zjQ6vB1P77c*EJgV_i$`NaYKI}*Oh)xXk*y(c6(IcxIwj^$9o{n3?F|v(B2RGV&d@K z7VyK)298VB+CnWX#?^-FSbl!;aqwjIX*~b#vds+kc?wRZF$Pd^R;(Kzofs zMqxi}3RbSQy@!>8rSda&`4~S19dzW1#0p&a#SOx2po0!L8WH5~PvH6Qo%Z_?P_rW6 z2;jtESHb6CAUpRq{w>sMDOed-2;dz@#?|xC9}b!WKm@Mbp>-NA#Ejk>*zUdlkKXNq z88%^xv9z$_U8*BCsb0{}ct3Y5bS6GBJ8s4E5A23}bGs3!;C&w+_c!cdoi9{h!>(ax zo#$g9P0AMIK06Pa<-CbnM-5kd2QZikNqa0J5{rKO`BPOdIv5=GIH8y7f(_0H44m_b z>ggA=iP_1im@v8IK6}US7XPAF+nPHHe_!yx0G5G~i2_5MhK}`ntTE2LHB200qXWJK zGu*ZRv7WJeS3=|ajhULksCjf-W8>d)L+m;knPfRAQau(+K0k&dFV;5kQE2DE9)OMZ z?tmL0a5WYI*3()tAVCVi$k1%uO7&^XlvV!M+p*>V%h`TQ7t|mbi5c(;eZ2!23{-?k zd$2O`QdFNq@!USAgo?u+B=#77MZKp~$Dqd88JRu;7=-rt|FxS2$e17ARkiAy4ze_rfGgPLdKTcf{qK+UjFb4Q@5%~D8 zN#XAcRa*Y1+V+gXZ(RRl1g?xss*AD9)blvb@;eO!7%H_Da=m}QdpYbWcFlYPGoQIu zvA!4tu7W*)XSFwER+Ou%`V<0pou3Jh0gD^gG3{g3LkPU=yG?aF26EezjmMTE#eDZN z)kN%Do{AbVx%*3cV8Q8dT!tm%*WStjuOo@^wzt-%sV|Mhz-PA6FS4dlcwqE51F<{U z9ZT>;U!z?xAnjLs9EDw;A4dS9d$SD43DS0)pY2C{*kfc0 z?ndA>aWw34H8kI8A#RV2@o1m1s@J!1X!|kF>&hLfTSnPDUd8>`3o8#QG#3)U{-QkM z1O2dMI>M?ooMX>n?Z=Ys+4tBzZZB(CB+&;}CYE=gWIvW9{9Hw~4T=DV%hH63hgG}G zRGnkDK{0CEG{J!7HC&H|?-(6NsAS;y*!KOK$5cn6wnRDt5>f%09%8@W8lNB~cJN<5 z40@zPkKcy@$?XXTcB$%C1XeZ=gQJTY2Rq)t``{QK9|H&aHr0%YhNESEUz|!bzNcbG zG#A|8HBlp~5dvM!QJdnlt*R~IcsIa#uDGONN7V5HFZ2)5*sNm+1{~9o*^~4IEutf^ zGHfou0C}$JC6&X&uW?}l#&tWJLP4}g|M)Csyen7}TUI7ss0K8#1X|v3hCS{L`*p_$ zc;eC^wa0jziwUAc_*L`}>xl|Jo8=c)uNd zFo0DwjP00zN<_zsVrcc6g!u0_41704a;xEYr|b&Uff%)}FrbND562Y&oKDvm|5h$E zuY<)5q+CDM2iL>e#iarQv7r*UG7Lp^7>HpABv<*T;11k&a*h5Lzr%Atbq8v7EkJSc z`S3JW!eQHof%-9X$;I71zVEvd9_K=<-4y<|7__I_$aFZpsp>@tAojp6kLMupR=I95 z2M&S8XeZ|(qpweG)vrwlkwxBZK_K1@N7FM9`k?Aj z1Yokx3iAeF4AiRO{vC_GtCjebe6I4NC{JhZfvkgFi4sd{A(a>kY_wm^;IG zdv=JP2E(AW!pu-BYlh7VEcwmw8J}+~fxGi;)mwWS|CSfp*I@~E$9x11;sGmgWo4SB zhuH5oI^4@)m@n>ExC7)EM;QZ;e?M1!45{XQa2E8j0f`xB>Q1Z-P&Q62ub zY9*`2U520inUgtsXvf*;hm&4bU5DbnvtcK{#0>oy3~Zi2p!z;!Xbc*O`v(J_OD}@r z_w_?NoPS}Mw~J+l{fFJVY)vt*TY&uPW9UzI{ieF~PSu~$p5H&D`T{bhhatdl2Qq1H zKxS-TOnCG~rd4lbs634OAa$y}_P8GcKV707YL9aadyI_bH3*PAj`Q6cfs(u0tA72W z>VNPvQ{x?CkMX@(52@}&AZs3e=gU}q?u0;F$5dKHvBf(*hBY#%C1`~6!YaC{e zjr(B5cKC1K+1ww&@9KpmWaWp@rSKG5Toj9)xpTXD|bP`=gBrl#^v3=Cg47qgkr)UV?eUkS_(9&j0e;< zInvf~rNYs-5&{;Yzx*7TBM+mN-j$dkj>gjD{9UQ+D(-7!c66w(dL}X)|Ml^EiJxvK zw4=!Ws!t%}ptaermvx4XuM7>4499dmcD4N*yWK8Ft);E7M+cA*n}%Ir({bFM=kP(# z8Q(k6cwF6hJkEGr33@zC8Uz9(dB~Kx05$rCVSpRC%FlMdNB@+Ent3}g)4d8yzLuaJ ztwlew6a9Yjqi7EZ=x5hD$@fk^$?K2$@aqFq>#{PIn%{=wjGo}*KC;^$qdg>I$=!zt z_8zF$DShEWB9{=Ui3niN#nRdK-3qoNf#Q5z*A@u8uBU6^7r(d7N2=$*?u(UB5E0WyffvOYK_95Ma8&VSfji7-@^k`WujWWerdeQEm)} zZ8TEV8zdSAEK~wlMj&n%W^#vNptH&hT)7aq3h{yUMOUg`XgO9}{4`)GDn4p^EJOeB zG?soWup&CA8~<-CQ4YRHL*WC?+~Mdi@$_m87zQJd^*na%soU7cz{4i$pa^QmFIE4& z*;$Tq$b$R8jR4Mo+f)bMg4d^(v7Pl1coqcCHX*?L8)jI4!BU|0=!cGdZycA!Up~IC ztnT;Y;ngfW2s~6rAaghd3=Q`hZ!9Z2U=H9BxH^`qx(FHgQxUNIV}a^k*z4Tl?$2CS z1MvWF(sBVZmM%iZk33}a_`-)hUV+jVPa{Ax5B-Xa4ljG0hjwhOa9RU<{0I8i5oo{l zGM)9aY9TZIVq`|IkTSYo$_isiV;o9a+z0=m(yjI!R{d*LJ0lZ#y^rH1rMu7iV#!{$ zE%qK(#yP6fk(ujx+~|*UL*)W+31JZ_IHM&QOPHowwJKf+uk+d#GsrE*SkD-{1C|(k zj@7{;a8&^V%IEQY9TP+J#HK;vNE|}P^C)J{=6_{6zTpwL5;(~HFtbo`i$jjfaFnNG ziOV23iVLh6^sIv4AGD5gQ3rtH-Ja7h(0d=(@lFf~x4mf$!g zFc7%06ty`v4|o!GrovQjdr)dS>tkE?ov`E2V>i>jf2ux({$?qb2p&7Xq(6|?xFopH z4sOB3*`+B(9Jm*k9Pa^t4gwyt&@OL6?ZNBN9zMfvHaij6%q^)06r}dMA2OCI|Kv=2 zJPS*x2H`r*!xFFjz*V@~WB4cOu*Vg8)S)(Zx?-Jnpmrz7Yso z-A;Amh^x2;gA%SI;c6-u|ZjVqEYM0>R?d>dI%iKTZjATsDpi5P-|dT5d)-~REJ{*<)@cT zM=eYOS5|v_SGDsfhumj|1Mmj|j}IUtG2SdD><0~d9EVJ-2Qd-19}_tFOJ@UZkB#xx83@GGy1?0|_b@W< z|LAB_KVjIZ^U=;*zCv44ItRu3;kg*6opqiw;t{hNsU8aX3&8+ZF)6sH`!;l_Ge(4o zLt`0y)Q0JafKv2>CH+7nbIWjmdSGeM)0lyc9%~GUgMC1e?L9Ed!sF4%-1*dMZ=YKZ z2ijqnIbLUWS8dVP=KCz$yWPdZQao`?b$1^h?UGB+ya$ZtL;zCkN26cKz6~!#+k3e6 zP)xthoxF5@1@}h;GC3OJ`bQujTxOXO7>FH2O^_$B1nf?H(&sdhu2|%{4SO{ry?WIXNiI^>)+47#BIxRHrrz(V{QjBh@QZb z(;e%5+{d)X-uBqIZnNQczXX3ftFbfeab2vusQX56oZdO`Ye!{tISfK?{v&$B`GN31v5o4EZoKb5_~a0b?$dQ2Alz=8g@Nu)QgwQubgb)TU>v|#{l&|>?$@G zOIRl$vorlv9|JTl`LmbuMe`SmOosP$&V#+3gpv@qVsnj^?R~VrOkNTZk#kXmzDjkQ z4+H_UY}DQwiwvS=PRR@lwt=}}kH1F1XqFk+w`oaXRSeJ@1xX-gH8TG;erB_I#Zu_* zsEJgL_d}_uhlGRfuiMnm7~k^`R_d&OGF+|z16XA<00S#-Q(g3Tc%6m@k4_cQ002yf zNklx7n1iCDPiHLSZEG2pYYw$1VRs~ zUc>W#*a_G3v^jYQ=MrdScSq2lu4)4N%bzrp+bs&xV?pN%p@-&J>AMNJtXovfZZ{-fE&Cj>_ zK;W>YVX51PaG;K3<|98-fCHV9k?#HQFVxmsHP3iofCnP3!>ZNB@Dc?D#%{qI(2w4S zO(6cs@!@dh^3;VSW~Osz%L`F`2?4C*KY9<`^EXo=UQ}$V+9KBG1Htu7hu!!P*Dv)A zZ+lI9TsHQ2$sMXIuD917i)D|e1(%;crwjU-b@m=sLrlb8yrtm#iTD8iK_vXZ%NF3Y z)w1b60DCS{oj=}(-3wV5!14=oF*CKHRd_Y14v6SC7+^hPE+sS`8Kh;Jh?yCoiFI9#ndmPtL;W&p+hrl+ zGk*diou{r)dkhcG%DlJR9KpAiT^_D0Rea>%G;=5f(ziY3aOGn{cBEJWhRw({Y2B^baeB?Si<4q&&MpSv3` zgm_9{Q>4nCyZ$8YEXKRzZoU)_CAP(_ZLPIWGu|Jp7=~;eQc)kq@7ht(_??{j^`aim zUlqGLXv;8R7rGhO!J&JK*Cr5lX$2gG9Gg?#EA8W@adiKRjF5B$l1E{w;QUcO?q>$B z!lPY)#doQW_!a%PRYS+hxZ(svk*JH>k5%oOkTnL1zh9`Dav!fvAnfrV_iIY_(a_DT^6b1#t>?e~KMu?{OT$E?5g&(clMi;JFh4-Lw>&k836`O)&T|-@K{% z^%frvJOfwZ(SP82Ct;vH0{+IHdyRgZ8CS)k<|epjt?WIHWMo(_s7HGoNPD~+8Cb{d zUPAhNaE=S(kl>JNbHCGYmFmTp8ZXd37DzNU`RIQ(%*(@g!KOg|W{fWv$J6m6Wr1+$Mp5;Pk)m3srSU$i_6W8`zaPc!uWwwq!3;XA)KUcI65udDiVkPN zCey$%a8($mU*5coZZ}&*EOK_7~`5_U3e}^GA>+=O`6vCwCQ*- zzKXyu=iQN^95srU%6nYq{qQvS{};Vryw9eURkN{+^8SBm=t-$C0uTu3^h~BIs*fSu-k_nM(ag^%*RE`qiAVj6l1g!&N9X zlWlTPjqh2948zn-ye5Gp8~sV_)eqob zcr1T}#fnR3k29QMkJn>~@X;x}CIPj_s7;$$jpw2yBx9T~Z<6sspeF8(@zsSGuQvXO z*QT)krUy#g)NgL@VWlGQIK$c`E+xpv&1Kb)Noheq!r0|#F$TW>n3+Z)IwM>WGh^M1 z4!R{me6M27ZmkcD0Y+h__xbok7jrWD! zc#<7ZKuEb#^~avx4pztw$c!7UGipyXM(v4_m;r6S-1xs>4P03>zL?R?ds=loYOqY0 zi04r|Ny;H`#qHXQMR}WxXJ=!^dFL6t7KAuuAmF)5^-E;f9H?EmJt85Bah*5*tor>- z`n!Iy#|Lb7yKvv{n9plLg14Vq*GzT&74`-qB2!iSysH{j#Teg&IcMR#ob#T&{*J}( zs9Bb>CETwJBdmdGc<8u(KTmZimJ+rI)!z%w9H#mfX3F|o0lU#gM;&aArQ5iFK7(Kl z`RJEbGqLN4R6yX0#bS#Gwn?6aC74MF6uO@XP;*A{_$<^iTg{n!LL9?ZY=SZEJ|BS! zX1Iozp9Ox#fNb~K_6`g~mDyNwl!Ei?9uuI;jM_oj$YfvDo7aK_FUd)&^RT&7N3nN2 z)DcC>8)Fyw?ceemiHP*V^{Ls(-s3p<3&um%%BHiz-QQsV%THLfjj;P3*oNH*XWXj# z@U#&9`3Trme?|4`vBv9GARsz8f0t=setlf^IXKS8Ucw8p3|y58Wbn;G$2`k={tugv z%c`YvoR=sdY=$QARFMBM=Xuo~R;En@1}egt!HE?BgSPPMv^XSzW zUk}3z6`>hy`lZ6D9Wx!}z;>VCp*!i$b3sJX53By(A5S#kISEUQB{?fURkeFbPRQca zB}KHyfwjj^VO;$f&qZ1IwQG@ywZ3!V{hQxFE#@e!jOkq!*SSWRj?=+84OQE0@-e>k zHwawiVPa<5pQ_o>0X`1L!T?r4JBWNo>!9ji&-2s-1x81)3N!u>V(H-akB#@G8M{&v zLP|`9UsWsi=eZyv@pZlXVM4nsA~F{lA!$GI)DYq?Vq2?LTF!G(7NoRSo!71K{>^U~ zftw3aqNBmj;XO|8i*a4$)d4;ZBXGInL)8WC3V&bt!2p(@DL9akj-_T(Q9~eyj<_>1 zGN`s;ZTQ28fUm!~XxEz%QgXZ0V4!N1+B`Kx1ra$qKy^{FO#?te!pR=L zuUeI7rc9ZK-O5uRwD&lgp>#(Fv&2o9>;hf4t!l4^#tVT87#VhJK9-d3Vw|Axj)?@8 zxgV%*XluN{ly`ecgT`W@G*0!=L2#6NnjMZ+^NgpCkWypxe-}NVT0WAeCYZ0m2+fP;`Qvd0}TksG^}r=~1fgQd@FtJvF*m3v#Y4|b1_v;r!5 zA$sD?47Am%XFOw$^8$NV8B|MF2qkf{0G;+s(q2^6tfDi2Wt0m zBT&2%OE{O^Z@e#;_X-B!JO#F3bV$E0Ro(v@4TCLYL6LEl*~izvUemB6aK%hH zLQ0PnxUzO#fg^GEwW{aTvB{w$j$#jHgm=Elb3y*czwO}vzrb@rNU4c32Flp=><(9rhbw~`4>)jZ&$hw)esiW;_2fw5CRZ?r-B1Jl3BtRH} zsSS-)=M1G`2q@OfeGDGnqxu$h-*^`@P3~WnGK%kMLX&QfjcPcD{exLFMLCRUbu0PDB#zahYNS1UFz@ zvgU!p-{&`QUCUwj=RV_c-5)M^{2D4;vPJb&n~S?0!EwxL&UirKFn|?cjHQH_>BzlE z^@sCR)446Mf3X5r*4k&RHU4CR>MICXWzVw)+PH2s*o2gwok8YfjoGThk$Lqt0?9Qk zhsml*SB|GLoE`i4zu{ls!c#*;QeRfxU6bd6kW$;@OqJ>=CjL|t)d{GHRO>3dFP-+- zpBzJ`+uZML8m}AIwd38Y^=}W&^IZ;pXa4~hmd`8L|L_||KyW2$eXdO{{C$K0EPF9B zc@|x-x?)e^?<1w`D2$?gKm4HjEY><7M{P2x;62bLgp@uIQ42F%H+`Y{BZ^dxS_B6g z&*ulOD)JQ4KM~ovil>D9k7U%QJop#S1tBH1$EP5}>V_fkhksFh`AOB9ryKUT0#9Ke zn7?23h|S}VW*@1ZTi}Z(rz+DsVjP+oTIad!VKsZ3urjeG zOqeiX!UXHtlow$@5Ma6FPqg1TJY%K8Q*cM{2QuCsdqee*wfjdUTpwn_5e5*7Jrfn# z3Xbt=)iWPe?S>4-zQ~|C8`~7e+-iJ)RWy@o&6oys7 zT655-@zxaq!UmJ=J*>UBzvm|U7{3x97n0bE%e$dYzgqR7v8pY3hRWvrf|ZMLeiy3l zu7npDxFY0#z)^PL{)l}Cek_(4)O}X936>h1`HgD3WvXp@t2P~>TDhx_@3Cf@tQs8A zJoV)NHIMnKNmi!Kmpms#Bx$&6)>}LmBz)|#U^mtZRbz?k^&g1}Y1yHxiF>xo6+D)wI0ewZM0^)2{! z2a1Il_iyQjCz|peC>#c`Yz6@nk9Q{4j-#s6u^R$wl$H=H^%ljVw_qSWbfxNs8OHk= zxFSxZgfp)eu3qTw(XUj*jBE{T(_0O-7U~>QZ3M@+AspKV=$Yz2hR4WQtbLhkRis{* z|E8FZc!c376H)-#s1bSSLY|AT#u!)}ItOO{7*7oeB72No#i9Nl=EJW_#%|2jUkUJWfEHM4 zdOkApti7%BOS0qz*Lk&823%+@k-fLz-|cU5P-AQcmJa6L!CRncFo5OE>(NQ&N7RJ5 z4VhTgpX4bj5=QNu{XJEOA+o*13|zUaKq5~a37i2fGM0mhKvN|cyo#s^R2@Yq8{VW^ z7Yi!a!A7VZ)flyI8$1Ymyc72L z64h!52vmaqS`Gnx7cyXElyN*mOL-n>kDZ0l9@oRqTZ1KqU9A_)&8YkZ0#-5Ks`gG$ z{S8aEJ$r)ly!Kk5TFYj8v1PjIpXfJ)0jwgz0G2~Q043Ku6KmB|XxNApUuoWmEbiuw zwL1z7e@Y+Kr+!qOSe*-a=z!eAphgoG9Q5jLjJ#h>?xu>M!=~O>~0+_ zU9N)(nx>z?4kJUU33lac)D-p@yXV$I#(za!QG-f;KFM(pG#+>6r3~wS4!hFL zMeWe89q>YAfDXV|LfRSkMLitX{yo8UKC2*5)EBki1QWjj3kVt+kMj^9+_#l&y?8Ky z8yOFIdw4F0h-as2 zrj@JC09IJr8kxC$8>?11V$*S;KU)jGX60$d`xv+?rePos zq2MKL2^a|fVy4W8Yv9Vd!K1t#+K;fU>sS=8%>KjbaT&NG1>h>&&_und+6wQx7rRzY zM(v8ZOI5!?G1ezgQ>r&=urV2r2q)dM^H`&vQW}KhxUPg{LB{>@hNv z&-h369%Lp@#>DPiY>V~{CNiHyrbV9=)l;$bsuBWR`GG4!oD9Y}X>(NP)U)Y0jB#mK z6g{sUBuiXtVJT#fiiQ6!zhT5%W?04f_p^zY>c$`rH|V&*e#LG(_hOfl`nMQ2V5kSQ ztlw#*dQQa*+z2c&Nz1StT?DSu*_6nM_mgS3PDxVjjG5c3e^Kp%-GLe)qtAt5x(;kyhz?gXd150p{u=K1};{e~U z;B+jdJqJs1XMJdl&m5LXbbb@nv#u}vclnz{6v3Z0*m!}PkQ5VUU^$e6GqE-z18deb z_TGTDJ3L^{b*fKXt$N4`Tv-uz2Cm5e@8^vBslhhYhq2^uMsL+`ky$km8CMOh+7}<5 zJND zfsYDU0@wR_;{~TJf!n3h55?On1iEHyj`zdAQLKGqC3~-5@WT$?pdhYmE0I z^-%5(&$E+0UiGbNs=JX|Zq>q)Gz0)icw+6ElI?96YX7EsBbNN#fsCuOuZ2VWmJh>I zj;Ayr6g88S*M$&5g3}&jfLjZ@@ZRvP>Yb0k9>ZQoT)=)mDC}{zah-`@NkfU0Iaqqu z8zpA!N=WJ20RA;L7fZa%S?8-!E7k4?Z8`w-SMK{%r;S$4oMzl#9=7e`!3->iQE=eO zqCX7%kf+1@EL9k$da+f_zAH3~fvd14*4v>fhm5K#VBmkkHrH=`rrP#G*pIio9c$C@ z5Ij;mp(4v6^a&I|Ka1#`t~L64eE3 z3x6LeDhyyb1O%>pH0bk@O8t6&)w;+av9^;A8;tXE4c1^cd&tLmB;nP6;eM!x`(+$z z1AK)d>YXrS6|vCUvCb!wr!=4-12_|@=xe5{z6L|ze~{`hT(={mc*+PNBq;1LN;uTn zt~v%2N8g~PWao=uc6-qtI|2jE^%3 zhS}*dI)u}bRjee4s zmm$My9tJQgaJ>%x=KcSTF*EGKu8`7^=adja9D$GG!{=;(J)Wo96Ghmi3hlA2_82u| zlcDiVRhJ_}WiA3DE3m@h!2S5gy($f0e_hw}T-pKxjcM?U=3sNL-sb%nV7lRWY9K&! z6!Zi{XtK(uYu&m9Q0%t66GifqvAd&Oa1wiL746R9C9?(Ny#H|(t~Ub4 zwa&8Vu&mwv+F_Ty`PK`8+Fh@l_xb5uRkg`qc>WrD?&FQfU|eF|R~3zZkdzPxuxtVW z7Hh@{1HcNX3GgC{OJ99n*&4V|ZMWx4K_FwQ>R0CZBEgN1ni$t1Bj@dVRqN%kpDU5Q zy-_tCHTKd^QB8ygvL6AbojX*wp}&g9KxG$}^6tY-U@{(OZZ*EB-bt=6?^i+y2}*l> zRlMpuD4N_Li}tuA=hgE%h6TuENkisF5(en|5U|*VOtx*uRkvb^#xB_1{VA$P@OtKt z#`m2;dmIYw*Ver?2iMt>C&-s`rv+e3%j1wOT-Jg;oYyf_n}*9r<#JAc}HNt6UM9V#?1GwE1-+u7!On3 zi~H&D3e{tW;7G^#_+r$o$}iem%IixAAtX5LaZhZH^7c!r`GG5I;2Ng2cHI~-XWs^U zJVZ4a6RrvHQ+H#+XcqzzJA0_^9|3z@P4&?Cu*cY@DDCUOTwmgB(%}!!TCI8+R`AHb z0qzeFxap3Jry3|3aiqMTzuySBo`ITN7k+NMFTgeRQjm!9d#}Qemy{F+umXg!yPz?1 zH3+G@&(u*ZhX}SP49}M9+2=Kl=9*(lL zv@F;HSH;Yv%tC*V{EO-y%)D+#JKu_0dE1aWzUzI}q%Yw>;Q1rSpvp$T%H6{FqIV1} zqQgK4Atb!)@!7CTZ)1r}qq`0NHdu zwX5zC1Q?RB8)Mcdu*c1PeBoSZ4ccSk`?@bgJ2{BNh`Kh7!&;rG+HR@p{6O8mpa(Yl zsaM6`%d-mo3}E@0!gm+!`GD#x7)Vwfh8IvXro|_ZZ!-m16Rev*uADtKh^dq zUb)=59udgIa`Ti0gM;bXruy(Ls=eZU7;vZ5g!5%6(a&Z!QQd;wGZ!F3Y5_8-*1+S~ z`wATCC-DD^eSBXBHWfk$Apx+*SRwErQak!q4|02qC3#yvS6ztX|Dze~F&^((3483z zFJH?hLhL8=E!8;qm;F1P#D6SnjQhXK`Kr}+s;2BN6Y=xN3yjR&Qb{Pn%nLJ^VrKOI2^eE)@Tu*tBOf&yAxoI`~Tvpt;PlKgK93X1B#8L4kLrP6ROfeuisR+Ab=+ay!%m7Rz&a=lr6~gv9R*v@ zc@Vn`$+-nP+Jq27h!gB_`BkvT$mFOy&*AnM_SBj{dh1%%%V2-+z~)FlElLr@eKN-JvaMY_391&?LbK>F?QGN3d7fDa={L@ZCINC;Kt6t z<_lkAsp^$*l%K@XgnyCBoqMufMDy4L2qAsRqsQw$6;8?7IBv`5a#*!4~AoPBbIR0KijxMgzh(>1Ot+Q1t zp$1m|46DoDc4Rrcdr>TMDAwE$S+BYh$M3OrV~Q;JcM?JfAx^f(QJ8?a1xu2uOm&z& zM(xvmIL=|H&2?8b)qk)HpgV*1n1mmH!e2cY=m~pcT-zGs;Wq7z_m#H)!1%h)KGlkw z0{lC3y&ul2uexU@+ny~AjT99GK&%_k=pe4fu6NI&NOO71@TPkYgo&Rp*J2}}(61d& zjiayAd#YWB8Sjg6#__Fw7Y;p^1b%#*YR1iYL#*-toXI7G5E6v;_^k6(d%WaKd%W&S z)xp@b=L1Y2rQbz+OvskERab1Y={~@?w<PgqJ=$_&oBs5RBITiJeYk->_qv_#cu z*mTMB0$%4*WIw|IRxvR$cZ?gQGaB-^8&zu}rO;YZ$YPKI1~Y7tyv-DIkc_HjK) zU@`8WK98!_X<+klWq3bak2Uspzoxoif${#xg8z>YLI?>)d)%wCYQ1?jA6L5f!~d{E z=k9~5^ZuqiCd4FjsOqeVxV^tJ?hk)s1f07dL#axQ5`LahYp)|}&Ysr3@b4{ijD(eS z*p%&`6ytpinEM$9u!@Kg=rCrW&OmYY@hIZf42Cs7MLCvd&=!nLw+@JmcWV&f4v!Q{ zMa*3EXlAnk$o0No6Op3)#D%IW_W5WxG*pBTLPCI8c)C3nIK&=f;`dO0)hD~EE?Z7} zOvnisSSs$ECgdzufoQ=NH>kN$}G8J_5(Q67p^=1M85qyJOwx5s!t zce!&kJ&-AP{^WukCK6cAMxdzHM4Jx4xL;QuRsFOFp7_sj;OQt6LI?>#T7Rfor?xZf z@#?LrAHi>vI)=YSf18jKK>|hAcGcOR!+rf3uLo-7iIEZ63mI6Ii<+@y9B1bhs?BZ< z@cA@~wlBld>h&u*{eF6jSTf6zVQF&e+O?70>-H0{e5ib;Uon6GDi8?eU%PcdxVA9wTt|?WwB!x)wg5 zBmu%rdmI!PyKMf2@$3Fk0X_~RbF=Lr)mC>H@ADNqOR9OYzs6j)zkLNQ(?D86!Ggo}7 zde$7{`g8Y)fDm0zwdL2&IG#OctInENM8iq~LlHe!^7 zbVxHmN))71iNOG+L%O>hrP2*bjFwb7he(Jrm>{hnCC!l9JAeQ0*Wdp==Y7xFIomnA z<9Y6QuIIk4&-M9yJfoWb)aIiFwZrltdN36bKSs2P?)i~15yL!fghhAtzE^rr)j_^9 zhLBGVu%_gZ!X?Uvl~`n2HiUGf$w^~j!IRv!c$C0MLCH^%Dia@j5T?yZa*rd6vN^84 zW|l?Gscqsw%mb~1zqR?)9Lk4&(z0Ba1ntVQZ^WKuj6zIUd>S|L;1`!R)9>fry^Q;C zfADpw0;2!fT{2EwShj103FRFJv5w{%S9o9{8ULyg5=MG@rd0SG%H>k-{LQ~TdGv;& z2n8dgQPB3y>(UcaNJF*V75gqEl~MuW-jU=sYTd6C;ebj8?p&lnZabUp2P`bv7nPs0NT9u!1}SR_Tp7lC``K2n8icrH`qGT zfWP<5noD+)MZTYH&|cazF~KO_1n*w!_-JJ8K*-L>8wXx3 zqoe-gyf5Aw2{zWtHuq%uRvb;TeU>hl8Ow#LZ#A=OAHCRpr(Aih#G2|7{FVgTQImC**(PAqQ>F_9_FjRvp}%bX|(}0>-mx#=`U47v~?8Uwbp1%R9Bv( zF`t8mO4>5?s`4hE1UDTV%~k^jdVR?VAazY}7A3Ccx+8z4%CNbH{S*grg5dd!`T5AR zC0Nv2Tux+OdZ6iwZrN4ZTi*of<*=p>V5Kknw*i;c*I@`A=Q69~+j`Wv$@jHPjC74z z&~Dk58x$>uoR-SX-K$7flqykCBtSiPtNUTevb3~IxVJHH=kV!k1qe&K+&kqw-EDoP zn~)kE-4#s%f%8B$DGtZcs4b_lzqNOxAZsI)!p+XfTiU6Wo4ilu zJCYzh1{ATK2Pyr}i{g(~Kw0dQ&U)4kxCC*>#~i7zuF)hPtl`(c<&Wferh}F*R>IP4NOk=;&PxeeFIvdtdYheXL>aS`f_9%Y6>^ADmdC@@NA;Tc|97skaV#*i=(NBxoA~5W}s;b3kaJC zumS7c__K@P7bedH=Ym<&N2scfDeDt)Z=!-V#_!l#KD;K}_2O>?=I8~xTGP(~aX{e z;c4bvuP{0f^|CVaprox#N6tlS_|a4wCCPL$I`Vh4asE>NBWZsIeqIaZm$SEu&2QId zJ5|RlJS`k<{V+ae<;@j{^FsYulni-aZP2c%;w96|7S=B#3@7HnX1%hTo1qBdl+WOJ zE(QGy%w85Qh@GDpNHLtHmhKh4ZiWx`2dypgRGY}`;zFK-BC5O*4m%>i?FBlS!Q5*Z z8#A}Axiicmr#Ca4<#IL>v1gYhGij4qqPmVEm*Fauk(Jf;{q<}AJ!GpC*3&naQE@|| z;q5KrmI?{CMXM2FwLr&E+_gZ3$6KB0Ru9xXi|d0w4)e?+h|G<)F}BMbzrE1;2B~6eVNgA8>GuU#x_e;j#?~cYo4$ z(yqBxCIzWI9_tU(sRplNVr`4-gEU(LuSLMa<#?~vS&_do<0rg_5AvgVnlhdRxLrow zXqWIC0HHNcK5QRI>)bJCpzWN*4f&+|cTLo-_~{_09-^>Inoj93&d&-FP%rKW2*qeG z<9BUNsRNNiBX{N)T#OW&bF(ZAZ^<{rH?h6(zB$aK1m^W)IA)_GAU7pM(ii`%`x!A? zqSoDa*49buls2=cG9Cq6s_u?SdD@i|m1E)=m!Sv8TM{55n;pl^Q;DBV^SAn-7K`a3 zNrN}fveO0f#U?<=FXu8p5&hIcw|QrD3UQp+_$j09F5>!k4|E$d0mJ*#bgEh^J6_LF z(I0WS^G|G5tHyQaU_EFUx2Ny8Kf)uw_E8xwb6d8Np=IK_n`bPnUECJpK{avj4K4dd z+?)y|K+KP+IkVKVEQsBCPBDgL;}Pu6G*l)6yyibx%?7nH|^D!?eOQBf_k~N0Ox6wu0z%};+ulPpdVcqR)b8S$hW1yUJ%mgMh6$5fwwaL3plF^GE85^Vz@khYF7K? zHw?E_Rv1Y~`r3H`f_~qm`oCkii}NDA7OAEej}%$B4x;ou+WVsE+9kM3XIo9!A{o)< zF0*i*y*TD$Aby!#c2&@k4$4w63cOZ5W3oAS-}0^VXz~7g<%w>c7K|FPf5hwT4O}d{ z#Xy=p)hpZ*^Tv7m2#(P>`g|i1xEI!TX@|09FRswzYp_Qkl(?BVQOb+b*vq`3R> z2~%ZVBGpS4Hc;4)29`BpY?bL|!tAr`L=)zL#&sjqbe5Yun42SHiw!YQEdH=_K2rUF zKFIcz@Tb2sy8#;}b*s8kSU(gorA+#~Oc7$f%f}hRI1a97E6?>#k_6O$5JaSwN}B~X z@iOds;ZEUeagWTZQ{EZ+3Qas2ua17!`TaKzW5K>Nc;@)wi1S)4`BZ|EuI3g=P1F&A zJPo%PbgVr;QDyW@05M)Deho2pJU;5A)^?OaWf@!sefW30)uTRuM zn@Ub(G@(DA?Mxhwr>>s}xz|?P*%3Wmx2v+Vi{F3mCeV$LCmZU9xsy%#F>$=G*0(ym zyRs_`z2eMlHtGHRZAz=Edr?T*LvCW_ZSyCS{k{4|os$DflL}qKT_-uE51CJhs6WHe z=hR+j;{lWOs;Vyses0`U-lYwO<7o>yccw6x2OTM~pUPTh-prjPJZ(AGLfvang>H2r zLZMk!G_i4zI*iyHu|LjgJ^y98;QKJ)N(?4ha$}^%rfDRAov;*u^uiYsT%&lj7_4IP zu3$@hq;u=y-N7C>kE*su3lrAPIJ9Y_!6Eu(Ze6l``AwKZ3SIHnv*Imf(*EyZ;$h*b zB#2!z=H0B@yF&`Kn97MDy4|^77cw&)OI69BCGzY>R(RNL=bMXJZcSx-qIMMS*hUi* zQ3acm3yg`48b{j-FFQabZ8k~a$Py(lb5Mw53FnvT2gB@SuLSZ^kwDVvey?PtJKDcj zB&#V&~<-7s&{h!UL~xQfBU> zCf%ehg#{C2NX^_nDvC`TiD_~ScZpiE!uJm=YC+i8Z4TQvW5J$8TE^+q+ zgd=*5mL*%}ZaK6lsnhHuBV6W4|1&q>+Wl9L!JmP_8Oz97O?qm?duw9ox%y)F4fh@{G^-3^{-J_TJIT8RGiP25K4GakQc z1z2Y4sZP=$r>vb3Uw?|d?Dy+dN2PQ+3y@@9Dp$*SmW#<=0AtF@`~E#9WaPE99X<)( zS%Lqt=@nQT(2ad`@`!P&wsUO0QJSqd4WAGP$Qz}Bw#~1DI-ihpmfZJtJHGJ3vC6U9 zrG5;aUwg1-!mdez5v?2#GRt+|J;NoMs~NMf8Fx`%4`bT;^rEi1u02o!f&+i3)tD2l z;CpJjrC+2E3x&1Ak#@JGDd$0UL*qDLA@fpdmf|gv)kq}&XLn%c$Hrh3KNYJ$X{Gmd z>-JPhD5N`{^PG(E9taA$l}BJ^xQ`j;j%f0+Op~Kc0&(aYU)&VcDso#Zl$;fqsFS&VN z&9r*8Lr8WF$~?LJ<}cXr+nMV0yJ$g!sG5f`&3T_o^DVOjCDa#jaX(2v$Ti+t!Pkaa z_Jz7RcF+@(BAoLM@T$j!NPm7=KBc~VZq>eHV+zdl9MV%I0!YU*1w6>SJSPTY96QK# zUGwh$aqtZEQG4SX#x9-RZtk!F{@2p@wv=b>RyQl?lw3(@HDuSCT<62&zy>- zI$ia9GQY+z)EFI@@odj56v!a+xPj5*27&E1?b%$n(Fw=-5qAbOHFfzoKZ>TtJw%k? zp}|=`#aYiIehKSsXEZFt#$|d@?nnU*Lu22+?A$$11jyW(0|)0vE#cy;x_m?=+;y#0 z1*`s;+6sAM9@F80o+P#4hi`|uNu5%kOO4$vmICB4FLW~hf_89hPkz#ET|F0`*cUJ4 zz8hs^7)jSF=bN{c2MlNl8G0J=Lv82sNiep~-ql8)KvFzU=0fZi;D;VCuHff#ZhR2l z#{_gytQF`2TXWv$?k7IP)vutxFC`IGcddS6H*nW?%w0`9mSrs0JjiRRUVl3jy&y&{ z8Rv9Xu*uI{WLE%?1$gHkK*D?LFZEjJ=WK-hp4CL0T3i#XVS|_^moGxeqke~G7i$et z9UGA~^Ke6beyP^CeVtLNMQ()GpMQ1p+!5!nNU{RTtjOQ8?e!*_Z6io}J=3pJ3~(gr z(U|&fRo|e^5hFVQ*3<0q84p9%-E~fNfgY6rG9T?lV+-PP>zv3&>^Yz>sjxE;iaig9 z5E@>0!E54RIQfEkeh&rN@hjwDbH24|@XlTaA6U)yDa}dOuOM3!`kG;0r?S~BEx&W_b~gv0GW3EKC_Id zn>|DyB3U9AzM|bcZ3sHfmU)*ouF}%&-dx*JU)(m&J>-;NsLKVeZY+ptxz|{Qlj9={ z#L|Ar&0M7Qt+lUuFXs59XN#W`fnd?%gg+#sxsAoBeii{o04fo(zZtfZ#|8!HGo$11 z>RaZ2%8U#my2VE3Gp4L}w)|a@BhY4aQe@wn=T5lVc&+rV$wq5T+3y0iFysdkPxD8) z@p!LUBymxm*wlVJPU3k}%&^L4(c_&fi{u7l2PFx-$LR0-!fK6+VL*LwV7%4q0>vb} z9Tw>udZ!$;mmIwEOE!Zpt7h=Tb!#$99nIvKT*|Ip-%lB*#SoEWMOQV=ONNmMEGW;p zAkk}H(LNv5A)>V-p0u!mLQY=OX5r^NHprHJsG<($mPo+I22Rf#d6n{|Ow2RP>}<}Jw!j+DPn9K2Qk)J* zk~dD11TQjIkx{%Ta(ZXy&0S_F|Bp}~d&d!oq3R@S;lWFtU^kxjp|0RNG*>e$mBJY=H)vAJApqxK*ZlY%@q?g`IS6Hz>z%tg{JUOrA`yH|$cDCb zk3Mi&tJ$IA``))XJ=tDe|JWeQbea_XK;5(VC2JTeAX7W;4O&>q9+Oh&$9F_X)w)6g=4Xq;^ z@q}6+us;$#RB;);!>_%T@V(@8a9*~^D2vGzYH??21(@gYy(Anh?>Z)DU#G~?aZ#0$ z+~m(okfwr#%-;8%XX}&d&P!a7H#$Fh5l0e6NsHqRJOra3Q>oLfJV{FeUI|z?7%G>k zHWR8)tcwec2hhzo*1sDM2oLhaehW2MP!dH2a88ptYIf|Vz?%V9@C&X7S_UJB-5x1+ z!#lU9UnE?7xEHR8{Fh;p-zB=Mu)&A+b`sLfaF=-KL zy9=r8EO^S@Ra<+)PN{6k9njzqAdj*pG!b=GIsUGzG1ZHU;MFgCG;I2Gtoox7D_}?S zVLS0v6f7hwVXV5H+EyA&9XP5O;ecZzN^qrN=rN`PQTHRNZxp~eei3))@ncr zRC3;B5i83z;&Q755-8?bu%IGU<`S@IjY_limrPRAEah+c&fMYoqQ6msN!+OY!dUhZ zy5)&3jQ#47h+cuPJ?IK-o}%;RapT8TmsBW+z3eofrOes^m#l6FreN?cUkt5bt9`)5 zcCa{SsPcn|vCHIBbowb8xL2#P`C`7jsyk{9UZu=@if*nR9-O8}uN1^PrK&LtdM5H@ zHZfH@`g@_hn0MW86V$N8{RS4jTn5O#0>f0?L)?SMic{~U_r&p=<|&sMajH+&dqrPH z+BL&O48}KI!4%4n!E4q4(sj(dNFz!v~h#v;^p>Hj#+<^nbze?f)H?em1 zT6_I5$Dy-`>zLKW!@{Y_Cx_EtZBvbG`k(GnqG7ifb+8ofg5ZtWI|8MU#YCztCx4u0 z$*AMe^{cJc)(N=F_mzN(GDqk2r`G+SN^2CdM;EEPI1LSK?s?74;4qD4YR8?90Yys? zU@yex;mPuN0E9lWh?Wh?w}P1%r);_yCDW1Wt>c|Fky%87B3{@Mz4e2-UC@5>*sB?G z!v$0$*8)Yk$5a#?*A>m>I~gOZ`Z`RC)OQgOwa{;T?}j?+YaWUy^=O{-dhmHI;~;}) zq=Fm{#BUp>FA8mpk$WZ&vC|i!&B(A;B+yJ~pl-d5v-eoGZlG={1ihBO`V~eaVcFel zSR2{`$A}+^-1-_%uM$2a`vp6fINrRjY;|@9ClY)EfN~yGS6I0cw%W8t3SM~Uk$F6B8VSQ}b$Z;z&ey&Q%w`_|L4GGvTYDQi+ zBRg!L+%t}ZXja7*rB@Zg3b3DbcR+hK(acZTp-kSJ6v?Y6)EbIp3B$>;7a2}xHj%?* zog(e@*PBRN`fxIZ2bMZ>{F=#eEYVg%g##itWq2-r2B`V=x?k4S+j4YQ;$XPwEUKH6 z4Yiu%r7bwJO^Kxqiqun2W`Z0-5hFIyJ39_h25O>|8%!N4AC=K9o