Merge master: fix zip-docs conflict

Conflict arose from #22066.
Merges up till d30c4a0

Author: dhimmel

appveyor.yml

@@ -20,12 +20,14 @@ environment:
   matrix:
 
     - CONDA_ROOT: "C:\\Miniconda3_64"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
       CONDA_PY: "36"
       CONDA_NPY: "113"
 
     - CONDA_ROOT: "C:\\Miniconda3_64"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
       PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
       CONDA_PY: "27"

asv_bench/benchmarks/io/csv.py

@@ -54,7 +54,14 @@ def time_frame_date_formatting(self):
         self.data.to_csv(self.fname, date_format='%Y%m%d')
 
 
-class ReadCSVDInferDatetimeFormat(object):
+class StringIORewind(object):
+
+    def data(self, stringio_object):
+        stringio_object.seek(0)
+        return stringio_object
+
+
+class ReadCSVDInferDatetimeFormat(StringIORewind):
 
     goal_time = 0.2
     params = ([True, False], ['custom', 'iso8601', 'ymd'])
@@ -66,10 +73,12 @@ def setup(self, infer_datetime_format, format):
                    'iso8601': '%Y-%m-%d %H:%M:%S',
                    'ymd': '%Y%m%d'}
         dt_format = formats[format]
-        self.data = StringIO('\n'.join(rng.strftime(dt_format).tolist()))
+        self.StringIO_input = StringIO('\n'.join(
+                                       rng.strftime(dt_format).tolist()))
 
     def time_read_csv(self, infer_datetime_format, format):
-        read_csv(self.data, header=None, names=['foo'], parse_dates=['foo'],
+        read_csv(self.data(self.StringIO_input),
+                 header=None, names=['foo'], parse_dates=['foo'],
                  infer_datetime_format=infer_datetime_format)
 
 
@@ -95,7 +104,7 @@ def time_skipprows(self, skiprows):
         read_csv(self.fname, skiprows=skiprows)
 
 
-class ReadUint64Integers(object):
+class ReadUint64Integers(StringIORewind):
 
     goal_time = 0.2
 
@@ -108,13 +117,13 @@ def setup(self):
         self.data2 = StringIO('\n'.join(arr.astype(str).tolist()))
 
     def time_read_uint64(self):
-        read_csv(self.data1, header=None, names=['foo'])
+        read_csv(self.data(self.data1), header=None, names=['foo'])
 
     def time_read_uint64_neg_values(self):
-        read_csv(self.data2, header=None, names=['foo'])
+        read_csv(self.data(self.data2), header=None, names=['foo'])
 
     def time_read_uint64_na_values(self):
-        read_csv(self.data1, header=None, names=['foo'],
+        read_csv(self.data(self.data1), header=None, names=['foo'],
                  na_values=self.na_values)
 
 
@@ -140,19 +149,20 @@ def time_thousands(self, sep, thousands):
         read_csv(self.fname, sep=sep, thousands=thousands)
 
 
-class ReadCSVComment(object):
+class ReadCSVComment(StringIORewind):
 
     goal_time = 0.2
 
     def setup(self):
         data = ['A,B,C'] + (['1,2,3 # comment'] * 100000)
-        self.s_data = StringIO('\n'.join(data))
+        self.StringIO_input = StringIO('\n'.join(data))
 
     def time_comment(self):
-        read_csv(self.s_data, comment='#', header=None, names=list('abc'))
+        read_csv(self.data(self.StringIO_input), comment='#',
+                 header=None, names=list('abc'))
 
 
-class ReadCSVFloatPrecision(object):
+class ReadCSVFloatPrecision(StringIORewind):
 
     goal_time = 0.2
     params = ([',', ';'], ['.', '_'], [None, 'high', 'round_trip'])
@@ -164,14 +174,14 @@ def setup(self, sep, decimal, float_precision):
         rows = sep.join(['0{}'.format(decimal) + '{}'] * 3) + '\n'
         data = rows * 5
         data = data.format(*floats) * 200  # 1000 x 3 strings csv
-        self.s_data = StringIO(data)
+        self.StringIO_input = StringIO(data)
 
     def time_read_csv(self, sep, decimal, float_precision):
-        read_csv(self.s_data, sep=sep, header=None, names=list('abc'),
-                 float_precision=float_precision)
+        read_csv(self.data(self.StringIO_input), sep=sep, header=None,
+                 names=list('abc'), float_precision=float_precision)
 
     def time_read_csv_python_engine(self, sep, decimal, float_precision):
-        read_csv(self.s_data, sep=sep, header=None, engine='python',
+        read_csv(self.data(self.StringIO_input), sep=sep, header=None, engine='python',
                  float_precision=None, names=list('abc'))
 
 
@@ -193,7 +203,7 @@ def time_convert_direct(self):
         read_csv(self.fname, dtype='category')
 
 
-class ReadCSVParseDates(object):
+class ReadCSVParseDates(StringIORewind):
 
     goal_time = 0.2
 
@@ -206,12 +216,14 @@ def setup(self):
                """
         two_cols = ['KORD,19990127'] * 5
         data = data.format(*two_cols)
-        self.s_data = StringIO(data)
+        self.StringIO_input = StringIO(data)
 
     def time_multiple_date(self):
-        read_csv(self.s_data, sep=',', header=None,
-                 names=list(string.digits[:9]), parse_dates=[[1, 2], [1, 3]])
+        read_csv(self.data(self.StringIO_input), sep=',', header=None,
+                 names=list(string.digits[:9]),
+                 parse_dates=[[1, 2], [1, 3]])
 
     def time_baseline(self):
-        read_csv(self.s_data, sep=',', header=None, parse_dates=[1],
+        read_csv(self.data(self.StringIO_input), sep=',', header=None,
+                 parse_dates=[1],
                  names=list(string.digits[:9]))

asv_bench/vbench_to_asv.py

@@ -6,6 +6,7 @@ dependencies:
   - Cython>=0.28.2
   - NumPy
   - flake8
+  - flake8-comprehensions
   - moto
   - pytest>=3.1
   - python-dateutil>=2.5.0

ci/environment-dev.yaml

@@ -10,42 +10,42 @@ if [ "$LINT" ]; then
 
     # pandas/_libs/src is C code, so no need to search there.
     echo "Linting *.py"
-    flake8 pandas --filename=*.py --exclude pandas/_libs/src
+    flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting *.py DONE"
 
     echo "Linting setup.py"
-    flake8 setup.py
+    flake8 setup.py --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting setup.py DONE"
 
     echo "Linting asv_bench/benchmarks/"
-    flake8 asv_bench/benchmarks/  --exclude=asv_bench/benchmarks/*.py --ignore=F811
+    flake8 asv_bench/benchmarks/  --exclude=asv_bench/benchmarks/*.py --ignore=F811,C405,C406,C408,C409,C410
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting asv_bench/benchmarks/*.py DONE"
 
     echo "Linting scripts/*.py"
-    flake8 scripts --filename=*.py
+    flake8 scripts --filename=*.py --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting scripts/*.py DONE"
 
     echo "Linting doc scripts"
-    flake8 doc/make.py doc/source/conf.py
+    flake8 doc/make.py doc/source/conf.py --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting doc scripts DONE"
 
     echo "Linting *.pyx"
-    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403
+    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C407,C411
     if [ $? -ne "0" ]; then
         RET=1
     fi
@@ -131,19 +131,6 @@ if [ "$LINT" ]; then
     fi
     echo "Check for non-standard imports DONE"
 
-    echo "Check for use of lists instead of generators in built-in Python functions"
-
-    # Example: Avoid `any([i for i in some_iterator])` in favor of `any(i for i in some_iterator)`
-    #
-    # Check the following functions:
-    # any(), all(), sum(), max(), min(), list(), dict(), set(), frozenset(), tuple(), str.join()
-    grep -R --include="*.py*" -E "[^_](any|all|sum|max|min|list|dict|set|frozenset|tuple|join)\(\[.* for .* in .*\]\)" pandas
-
-    if [ $? = "0" ]; then
-        RET=1
-    fi
-    echo "Check for use of lists instead of generators in built-in Python functions DONE"
-
     echo "Check for incorrect sphinx directives"
     SPHINX_DIRECTIVES=$(echo \
        "autosummary|contents|currentmodule|deprecated|function|image|"\

ci/lint.sh

@@ -9,6 +9,7 @@ dependencies:
   - fastparquet
   - feather-format
   - flake8=3.4.1
+  - flake8-comprehensions
   - gcsfs
   - html5lib
   - ipython