From 3efa4983daccba911e0a2390ba1df54e66cf2e83 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Tue, 22 Oct 2019 23:39:40 +0100 Subject: [PATCH 1/7] Support for patch version in min_deps_check.py --- ci/min_deps_check.py | 72 ++++++++++++++++----------- ci/requirements/py36-min-all-deps.yml | 2 +- ci/requirements/py36.yml | 4 +- ci/requirements/py37-windows.yml | 6 +-- ci/requirements/py37.yml | 4 +- doc/whats-new.rst | 2 +- 6 files changed, 52 insertions(+), 38 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 3bdd48ca76d..a5ba90679b7 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -6,7 +6,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta -from typing import Dict, Iterator, Tuple +from typing import Dict, Iterator, Optional, Tuple import yaml @@ -34,10 +34,14 @@ def error(msg: str) -> None: print("ERROR:", msg) -def parse_requirements(fname) -> Iterator[Tuple[str, int, int]]: +def warning(msg: str) -> None: + print("WARNING:", msg) + + +def parse_requirements(fname) -> Iterator[Tuple[str, int, int, Optional[int]]]: """Load requirements/py36-min-all-deps.yml - Yield (package name, major version, minor version) + Yield (package name, major version, minor version, [patch version]) """ global has_errors @@ -52,15 +56,18 @@ def parse_requirements(fname) -> Iterator[Tuple[str, int, int]]: if pkg.endswith("<") or pkg.endswith(">") or eq != "=": error("package should be pinned with exact version: " + row) continue + try: - major, minor = version.split(".") - except ValueError: - error("expected major.minor (without patch): " + row) - continue - try: - yield pkg, int(major), int(minor) + version_tup = tuple(int(x) for x in version.split(".")) except ValueError: - error("failed to parse version: " + row) + raise ValueError("non-numerical version: " + row) + + if len(version_tup) == 2: + yield (pkg, *version_tup, None) # type: ignore + elif len(version_tup) == 3: + yield (pkg, *version_tup) # type: ignore + else: + raise ValueError("expected major.minor or major.minor.patch: " + row) def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: @@ -80,9 +87,9 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: label = label.strip() if label == "file name": value = value.strip()[len(pkg) :] - major, minor = value.split("-")[1].split(".")[:2] - major = int(major) - minor = int(minor) + smajor, sminor = value.split("-")[1].split(".")[:2] + major = int(smajor) + minor = int(sminor) if label == "timestamp": assert major is not None assert minor is not None @@ -109,17 +116,15 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: def process_pkg( - pkg: str, req_major: int, req_minor: int -) -> Tuple[str, int, int, str, int, int, str, str]: + pkg: str, req_major: int, req_minor: int, req_patch: Optional[int] +) -> Tuple[str, str, str, str, str, str]: """Compare package version from requirements file to available versions in conda. Return row to build pandas dataframe: - package name - - major version in requirements file - - minor version in requirements file + - major.minor.[patch] version in requirements file - publication date of version in requirements file (YYYY-MM-DD) - - major version suggested by policy - - minor version suggested by policy + - major.minor version suggested by policy - publication date of version suggested by policy (YYYY-MM-DD) - status ("<", "=", "> (!)") """ @@ -130,7 +135,7 @@ def process_pkg( req_published = versions[req_major, req_minor] except KeyError: error("not found in conda: " + pkg) - return pkg, req_major, req_minor, "-", 0, 0, "-", "(!)" + return pkg, fmt_version(req_major, req_minor, req_patch), "-", "-", "-", "(!)" policy_months = POLICY_MONTHS.get(pkg, POLICY_MONTHS_DEFAULT) policy_published = datetime.now() - timedelta(days=policy_months * 30) @@ -153,30 +158,39 @@ def process_pkg( else: status = "=" + if req_patch is not None: + warning("patch version should not appear in requirements file: " + pkg) + status += " (w)" + return ( pkg, - req_major, - req_minor, + fmt_version(req_major, req_minor, req_patch), req_published.strftime("%Y-%m-%d"), - policy_major, - policy_minor, + fmt_version(policy_major, policy_minor), policy_published_actual.strftime("%Y-%m-%d"), status, ) +def fmt_version(major: int, minor: int, patch: int = None) -> str: + if patch is None: + return f"{major}.{minor}" + else: + return f"{major}.{minor}.{patch}" + + def main() -> None: fname = sys.argv[1] with ThreadPoolExecutor(8) as ex: futures = [ - ex.submit(process_pkg, pkg, major, minor) - for pkg, major, minor in parse_requirements(fname) + ex.submit(process_pkg, pkg, major, minor, patch) + for pkg, major, minor, patch in parse_requirements(fname) ] rows = [f.result() for f in futures] - print("Package Required Policy Status") - print("------------- ----------------- ----------------- ------") - fmt = "{:13} {:>1d}.{:<2d} ({:10}) {:>1d}.{:<2d} ({:10}) {}" + print("Package Required Policy Status") + print("------------- -------------------- -------------------- ------") + fmt = "{:13} {:7} ({:10}) {:7} ({:10}) {}" for row in rows: print(fmt.format(*row)) diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index bbc51d09ce2..08ea4959b06 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -13,7 +13,7 @@ dependencies: - cartopy=0.17 - cdms2=3.1 - cfgrib=0.9 - - cftime=1.0.3 + - cftime=1.0.3 # FIXME incompatibility with 1.0.4 - coveralls - dask=1.2 - distributed=1.27 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index 54ab9b5be7a..9d1d58ce149 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime=1.0.3.4 + - cftime<1.0.4 # FIXME incompatibility with 1.0.4 - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy + - numpy<1.18 # FIXME incompatibility with 1.18 - pandas - pint - pip diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 3318d837257..8679431e26d 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -8,8 +8,8 @@ dependencies: - bottleneck - cartopy # - cdms2 # Not available on Windows - # - cfgrib>=0.9.2 # Causes Python interpreter crash on Windows - - cftime=1.0.3.4 + # - cfgrib # Causes Python interpreter crash on Windows + - cftime<1.0.4 # FIXME incompatibility with 1.0.4 - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy + - numpy<1.18 # FIXME incompatibility with 1.18 - pandas - pint - pip diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 5cdb634649c..38292e31332 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime=1.0.3.4 + - cftime<1.0.4 # FIXME incompatibility with 1.0.4 - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy + - numpy<1.18 # FIXME incompatibility with 1.18 - pandas - pint - pip diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 2fbfd58ddba..0f4d0c10f1f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -42,7 +42,7 @@ Bug fixes but cloudpickle isn't (:issue:`3401`) by `Rhys Doyle `_ - Sync with cftime by removing `dayofwk=-1` for cftime>=1.0.4. - By `Anderson Banihirwe `_. + By `Anderson Banihirwe `_. Documentation From 7621d34f1bd38703bd24cac0724d519c2cd4c7b8 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Tue, 22 Oct 2019 23:50:32 +0100 Subject: [PATCH 2/7] Fix requirements --- ci/azure/install.yml | 2 +- ci/requirements/py36.yml | 4 ++-- ci/requirements/py37-windows.yml | 4 ++-- ci/requirements/py37.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index f1192f4424f..2c78bc79aa8 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -15,7 +15,7 @@ steps: --no-deps \ --pre \ --upgrade \ - numpy \ + numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 \ matplotlib \ pandas \ scipy diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index 9d1d58ce149..c1a7404d655 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime<1.0.4 # FIXME incompatibility with 1.0.4 + - cftime<1.0.4 # FIXME - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy<1.18 # FIXME incompatibility with 1.18 + - numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 - pandas - pint - pip diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 8679431e26d..5a0eca7380d 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -9,7 +9,7 @@ dependencies: - cartopy # - cdms2 # Not available on Windows # - cfgrib # Causes Python interpreter crash on Windows - - cftime<1.0.4 # FIXME incompatibility with 1.0.4 + - cftime<1.0.4 # FIXME - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy<1.18 # FIXME incompatibility with 1.18 + - numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 - pandas - pint - pip diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 38292e31332..15032f74c0c 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime<1.0.4 # FIXME incompatibility with 1.0.4 + - cftime<1.0.4 # FIXME - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy<1.18 # FIXME incompatibility with 1.18 + - numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 - pandas - pint - pip From 111d8ebe7bdbdbead0fe623857f4a13d631627c3 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Tue, 22 Oct 2019 23:53:20 +0100 Subject: [PATCH 3/7] Fix requirements --- ci/azure/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 2c78bc79aa8..af2584ea396 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -15,7 +15,7 @@ steps: --no-deps \ --pre \ --upgrade \ - numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 \ + \ # numpy \ # FIXME https://github.com/pydata/xarray/issues/3409 matplotlib \ pandas \ scipy From f172cdf6f7d04165dc601e09bc7b1accd5ebd379 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 23 Oct 2019 00:13:02 +0100 Subject: [PATCH 4/7] cftime is 1.0.4 incompatible --- ci/azure/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index af2584ea396..3503fd4efb6 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -25,7 +25,7 @@ steps: git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ - git+https://github.com/Unidata/cftime + # git+https://github.com/Unidata/cftime # FIXME condition: eq(variables['UPSTREAM_DEV'], 'true') displayName: Install upstream dev dependencies From de54bc784cb02a0f855b85766e42e0f8a01950e9 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 23 Oct 2019 00:21:24 +0100 Subject: [PATCH 5/7] Use cftime PR#127 --- ci/azure/install.yml | 3 ++- ci/requirements/py36-min-all-deps.yml | 2 +- ci/requirements/py36.yml | 2 +- ci/requirements/py37-windows.yml | 2 +- ci/requirements/py37.yml | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 3503fd4efb6..cdc1d35de72 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -25,7 +25,8 @@ steps: git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ - # git+https://github.com/Unidata/cftime # FIXME + # git+https://github.com/Unidata/cftime # FIXME PR not merged yet + git+https://github.com/Unidata/cftime.git@refs/pull/127/merge condition: eq(variables['UPSTREAM_DEV'], 'true') displayName: Install upstream dev dependencies diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 08ea4959b06..c99ae39e5d9 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -13,7 +13,7 @@ dependencies: - cartopy=0.17 - cdms2=3.1 - cfgrib=0.9 - - cftime=1.0.3 # FIXME incompatibility with 1.0.4 + - cftime=1.0.3 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask=1.2 - distributed=1.27 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index c1a7404d655..6e27cea2ffe 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime<1.0.4 # FIXME + - cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask - distributed diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 5a0eca7380d..7027fc11ab7 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -9,7 +9,7 @@ dependencies: - cartopy # - cdms2 # Not available on Windows # - cfgrib # Causes Python interpreter crash on Windows - - cftime<1.0.4 # FIXME + - cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask - distributed diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 15032f74c0c..a4c974c0176 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime<1.0.4 # FIXME + - cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask - distributed From 93ed663b818169731ee58692ad701d318e4d8d5f Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 23 Oct 2019 00:21:55 +0100 Subject: [PATCH 6/7] trivial --- ci/azure/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index cdc1d35de72..bfaf54fa4a9 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -25,7 +25,7 @@ steps: git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ - # git+https://github.com/Unidata/cftime # FIXME PR not merged yet + \ # git+https://github.com/Unidata/cftime # FIXME PR not merged yet git+https://github.com/Unidata/cftime.git@refs/pull/127/merge condition: eq(variables['UPSTREAM_DEV'], 'true') displayName: Install upstream dev dependencies From 359670a4b40e1fb481411fa111ffb0275bf63f17 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 23 Oct 2019 00:34:36 +0100 Subject: [PATCH 7/7] trivial --- ci/azure/install.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index bfaf54fa4a9..2911e227172 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -15,18 +15,18 @@ steps: --no-deps \ --pre \ --upgrade \ - \ # numpy \ # FIXME https://github.com/pydata/xarray/issues/3409 matplotlib \ pandas \ scipy + # numpy \ # FIXME https://github.com/pydata/xarray/issues/3409 pip install \ --no-deps \ --upgrade \ git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ - \ # git+https://github.com/Unidata/cftime # FIXME PR not merged yet git+https://github.com/Unidata/cftime.git@refs/pull/127/merge + # git+https://github.com/Unidata/cftime # FIXME PR 127 not merged yet condition: eq(variables['UPSTREAM_DEV'], 'true') displayName: Install upstream dev dependencies